Percent | Source code & Disassembly of vmlinux for cycles (2925916 samples, percent: local period) -------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001071e6f8 : : arm_smmu_cmdq_issue_cmdlist(): : * insert their own list of commands then all of the commands from one : * CPU will appear before any of the commands from the other CPU. : */ : static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, : u64 *cmds, int n, bool sync) : { 0.00 : ffff80001071e6f8: stp x29, x30, [sp, #-496]! 0.00 : ffff80001071e6fc: mov x29, sp 0.00 : ffff80001071e700: add x5, x29, #0xcf 0.02 : ffff80001071e704: stp x25, x26, [sp, #64] 0.00 : ffff80001071e708: and x26, x5, #0xffffffffffffffc0 0.01 : ffff80001071e70c: stp x23, x24, [sp, #48] 0.01 : ffff80001071e710: stp x27, x28, [sp, #80] : u64 cmd_sync[CMDQ_ENT_DWORDS]; : u32 prod; : unsigned long flags; : bool owner; : struct arm_smmu_cmdq *cmdq = &smmu->cmdq; : struct arm_smmu_ll_queue llq = { 0.00 : ffff80001071e714: add x23, x26, #0x80 : .max_n_shift = cmdq->q.llq.max_n_shift, 0.00 : ffff80001071e718: add x27, x0, #0x40 : { 0.01 : ffff80001071e71c: stp x19, x20, [sp, #16] 0.01 : ffff80001071e720: stp x21, x22, [sp, #32] 0.00 : ffff80001071e724: mov x28, x0 : struct arm_smmu_ll_queue llq = { 0.01 : ffff80001071e728: stp xzr, xzr, [x23, #16] : { 0.00 : ffff80001071e72c: adrp x0, ffff800011909000 : struct arm_smmu_ll_queue llq = { 0.01 : ffff80001071e730: stp xzr, xzr, [x23, #32] : { 0.00 : ffff80001071e734: add x0, x0, #0x908 : struct arm_smmu_ll_queue llq = { 0.01 : ffff80001071e738: stp xzr, xzr, [x23, #48] 0.01 : ffff80001071e73c: stp xzr, xzr, [x23, #64] : .max_n_shift = cmdq->q.llq.max_n_shift, 0.01 : ffff80001071e740: ldr w19, [x27, #64] : { 0.04 : ffff80001071e744: str x1, [x29, #96] 0.00 : ffff80001071e748: and w1, w3, #0xff : }, head = llq; 0.01 : ffff80001071e74c: str w19, [x26, #192] : { 0.01 : ffff80001071e750: str w1, [x29, #108] 0.00 : ffff80001071e754: ldr x1, [x0] 0.01 : ffff80001071e758: str x1, [x29, #488] 0.00 : ffff80001071e75c: mov x1, #0x0 // #0 0.00 : ffff80001071e760: str w2, [x29, #104] : }, head = llq; 0.00 : ffff80001071e764: mov x1, x23 : struct arm_smmu_ll_queue llq = { 0.00 : ffff80001071e768: stp xzr, xzr, [x26, #128] : }, head = llq; 0.00 : ffff80001071e76c: mov x2, #0x80 // #128 : struct arm_smmu_ll_queue llq = { 0.01 : ffff80001071e770: stp xzr, xzr, [x23, #80] : }, head = llq; 0.00 : ffff80001071e774: mov x0, x26 : struct arm_smmu_ll_queue llq = { 0.01 : ffff80001071e778: stp xzr, xzr, [x23, #96] 0.01 : ffff80001071e77c: stp xzr, xzr, [x23, #112] : }, head = llq; 0.00 : ffff80001071e780: bl ffff800010cb7dc0 <__memcpy> : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e784: mrs x8, daif 0.00 : ffff80001071e788: mov x21, x8 : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e78c: and w0, w8, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff80001071e790: cbnz w0, ffff80001071e79c : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e794: mov x0, #0x60 // #96 0.06 : ffff80001071e798: msr daifset, #0x2 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.24 : ffff80001071e79c: ldr x0, [x28, #64] : queue_has_space(): : prod = Q_IDX(q, q->prod); 0.01 : ffff80001071e7a0: mov w1, #0x1 // #1 7.22 : ffff80001071e7a4: ldr w2, [x29, #104] 0.00 : ffff80001071e7a8: lsl w19, w1, w19 : arm_smmu_cmdq_issue_cmdlist(): : int ret = 0; : : /* 1. Allocate some space in the queue */ : local_irq_save(flags); : llq.val = READ_ONCE(cmdq->q.llq.val); 0.00 : ffff80001071e7ac: str x0, [x26, #128] : u64 old; : : while (!queue_has_space(&llq, n + sync)) { : local_irq_restore(flags); : if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq)) : dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); 0.00 : ffff80001071e7b0: adrp x1, ffff800010ed9000 0.00 : ffff80001071e7b4: ldr w0, [x29, #108] : queue_has_space(): : prod = Q_IDX(q, q->prod); 0.00 : ffff80001071e7b8: sub w22, w19, #0x1 : arm_smmu_cmdq_issue_cmdlist(): : dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); 0.00 : ffff80001071e7bc: add x1, x1, #0x458 : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071e7c0: orr w20, w22, w19 : arm_smmu_cmdq_issue_cmdlist(): : dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); 0.00 : ffff80001071e7c4: add x24, x1, #0x20 0.00 : ffff80001071e7c8: add w25, w0, w2 0.00 : ffff80001071e7cc: nop : while (!queue_has_space(&llq, n + sync)) { 0.13 : ffff80001071e7d0: ldp w1, w4, [x26, #128] : queue_has_space(): : cons = Q_IDX(q, q->cons); 0.01 : ffff80001071e7d4: and w0, w4, w22 : prod = Q_IDX(q, q->prod); 0.00 : ffff80001071e7d8: and w3, w1, w22 : space = (1 << q->max_n_shift) - (prod - cons); 0.00 : ffff80001071e7dc: add w2, w0, w19 : if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons)) 0.06 : ffff80001071e7e0: eor w9, w4, w1 : space = (1 << q->max_n_shift) - (prod - cons); 0.00 : ffff80001071e7e4: tst w9, w19 0.00 : ffff80001071e7e8: sub w2, w2, w3 0.00 : ffff80001071e7ec: sub w0, w0, w3 0.00 : ffff80001071e7f0: csel w0, w0, w2, ne // ne = any : arm_smmu_cmdq_issue_cmdlist(): : while (!queue_has_space(&llq, n + sync)) { 0.00 : ffff80001071e7f4: cmp w0, w25 0.00 : ffff80001071e7f8: b.cc ffff80001071e97c // b.lo, b.ul, b.last : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071e7fc: and w3, w1, w20 : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.00 : ffff80001071e800: and w0, w1, #0x80000000 : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071e804: add w3, w3, w25 : arm_smmu_cmdq_issue_cmdlist(): : : head.cons = llq.cons; : head.prod = queue_inc_prod_n(&llq, n + sync) | : CMDQ_PROD_OWNED_FLAG; : : old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val); 0.11 : ffff80001071e808: ldr x1, [x23] : queue_inc_prod_n(): : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.00 : ffff80001071e80c: and w3, w3, w20 0.00 : ffff80001071e810: orr w3, w3, w0 : arm_smmu_cmdq_issue_cmdlist(): : head.prod = queue_inc_prod_n(&llq, n + sync) | 0.00 : ffff80001071e814: orr w0, w3, #0x80000000 : head.cons = llq.cons; 0.01 : ffff80001071e818: stp w0, w4, [x26] : old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val); 0.00 : ffff80001071e81c: ldr x2, [x26] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001071e820: b ffff80001071e84c 0.16 : ffff80001071e824: b ffff80001071e84c : __lse__cmpxchg_case_64(): : } : : __CMPXCHG_CASE(w, b, , 8, ) : __CMPXCHG_CASE(w, h, , 16, ) : __CMPXCHG_CASE(w, , , 32, ) : __CMPXCHG_CASE(x, , , 64, ) 0.00 : ffff80001071e828: mov x0, x27 0.00 : ffff80001071e82c: mov x4, x1 0.00 : ffff80001071e830: cas x4, x2, [x27] 28.61 : ffff80001071e834: mov x0, x4 : arm_smmu_cmdq_issue_cmdlist(): : if (old == llq.val) 0.00 : ffff80001071e838: ldr x1, [x23] 0.00 : ffff80001071e83c: cmp x1, x0 0.00 : ffff80001071e840: b.eq ffff80001071e85c // b.none : break; : : llq.val = old; 0.03 : ffff80001071e844: str x0, [x23] : do { 0.00 : ffff80001071e848: b ffff80001071e7d0 : __ll_sc__cmpxchg_case_64(): : * constraint for 32 bit operations. : */ : __CMPXCHG_CASE(w, b, , 8, , , , , K) : __CMPXCHG_CASE(w, h, , 16, , , , , K) : __CMPXCHG_CASE(w, , , 32, , , , , K) : __CMPXCHG_CASE( , , , 64, , , , , L) 0.00 : ffff80001071e84c: b ffff8000107217b0 : arm_smmu_cmdq_issue_cmdlist(): : if (old == llq.val) 0.00 : ffff80001071e850: ldr x1, [x23] 0.00 : ffff80001071e854: cmp x1, x0 0.00 : ffff80001071e858: b.ne ffff80001071e844 // b.any : } while (1); : owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG); 0.02 : ffff80001071e85c: ldr w25, [x23] : head.prod &= ~CMDQ_PROD_OWNED_FLAG; 0.05 : ffff80001071e860: and w0, w3, #0x7fffffff 0.01 : ffff80001071e864: str w0, [x26] : llq.prod &= ~CMDQ_PROD_OWNED_FLAG; 0.00 : ffff80001071e868: and w12, w25, #0x7fffffff : arm_smmu_cmdq_write_entries(): : for (i = 0; i < n; ++i) { 0.00 : ffff80001071e86c: ldr w0, [x29, #104] : arm_smmu_cmdq_issue_cmdlist(): : llq.prod &= ~CMDQ_PROD_OWNED_FLAG; 0.00 : ffff80001071e870: str w12, [x23] : arm_smmu_cmdq_write_entries(): : .max_n_shift = cmdq->q.llq.max_n_shift, 0.01 : ffff80001071e874: ldr w2, [x27, #64] : for (i = 0; i < n; ++i) { 0.00 : ffff80001071e878: cmp w0, #0x0 0.00 : ffff80001071e87c: b.le ffff80001071e8ec : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071e880: mov w13, #0x1 // #1 0.00 : ffff80001071e884: sub w10, w0, #0x1 0.00 : ffff80001071e888: lsl w0, w13, w2 0.00 : ffff80001071e88c: sub w11, w0, #0x1 0.00 : ffff80001071e890: orr w11, w11, w0 0.00 : ffff80001071e894: ldr x24, [x29, #96] 0.00 : ffff80001071e898: and w1, w12, w11 0.00 : ffff80001071e89c: add w10, w10, w1 0.00 : ffff80001071e8a0: b ffff80001071e8ac 0.00 : ffff80001071e8a4: ldr w2, [x27, #64] 0.00 : ffff80001071e8a8: add w1, w1, #0x1 : arm_smmu_cmdq_write_entries(): : queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS); 0.01 : ffff80001071e8ac: ldr x3, [x27, #160] 0.00 : ffff80001071e8b0: lsl w0, w13, w2 0.00 : ffff80001071e8b4: sub w0, w0, #0x1 0.00 : ffff80001071e8b8: ldr x4, [x27, #136] 0.00 : ffff80001071e8bc: and w0, w0, w11 : queue_write(): : *dst++ = cpu_to_le64(*src++); 0.00 : ffff80001071e8c0: ldr x9, [x24] : arm_smmu_cmdq_write_entries(): : queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS); 0.00 : ffff80001071e8c4: lsl x2, x3, #3 0.00 : ffff80001071e8c8: and w0, w0, w1 0.00 : ffff80001071e8cc: add x24, x24, #0x10 : for (i = 0; i < n; ++i) { 0.01 : ffff80001071e8d0: cmp w1, w10 : queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS); 0.00 : ffff80001071e8d4: mul x0, x0, x2 0.00 : ffff80001071e8d8: add x2, x4, x0 : queue_write(): : *dst++ = cpu_to_le64(*src++); 0.01 : ffff80001071e8dc: str x9, [x4, x0] 0.03 : ffff80001071e8e0: ldur x0, [x24, #-8] 0.00 : ffff80001071e8e4: str x0, [x2, #8] : arm_smmu_cmdq_write_entries(): : for (i = 0; i < n; ++i) { 0.00 : ffff80001071e8e8: b.ne ffff80001071e8a4 // b.any : arm_smmu_cmdq_issue_cmdlist(): : /* : * 2. Write our commands into the queue : * Dependency ordering from the cmpxchg() loop above. : */ : arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); : if (sync) { 0.00 : ffff80001071e8ec: ldr w0, [x29, #108] 0.00 : ffff80001071e8f0: cbnz w0, ffff80001071ec80 : */ : arm_smmu_cmdq_shared_lock(cmdq); : } : : /* 3. Mark our slots as valid, ensuring commands are visible first */ : dma_wmb(); 0.00 : ffff80001071e8f4: dmb oshst : arm_smmu_cmdq_set_valid_map(): : __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true); 0.33 : ffff80001071e8f8: ldr w3, [x26] 0.00 : ffff80001071e8fc: add x5, x28, #0x100 0.00 : ffff80001071e900: ldr w0, [x28, #128] 0.00 : ffff80001071e904: mov w4, #0x1 // #1 : arm_smmu_cmdq_issue_cmdlist(): : arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod); 0.00 : ffff80001071e908: ldr w24, [x26, #128] : arm_smmu_cmdq_set_valid_map(): : __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true); 0.00 : ffff80001071e90c: mov x1, x5 0.00 : ffff80001071e910: str x5, [x29, #96] 0.00 : ffff80001071e914: mov w2, w24 0.00 : ffff80001071e918: bl ffff80001071e4e8 <__arm_smmu_cmdq_poll_set_valid_map.isra.33> : arm_smmu_cmdq_issue_cmdlist(): : : /* 4. If we are the owner, take control of the SMMU hardware */ : if (owner) { 0.01 : ffff80001071e91c: tbnz w25, #31, ffff80001071eae0 : __read_once_size(): 0.00 : ffff80001071e920: ldr w0, [x28, #264] : arm_smmu_cmdq_issue_cmdlist(): : /* a. Wait for previous owner to finish */ : atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod); 0.00 : ffff80001071e924: add x1, x28, #0x108 0.00 : ffff80001071e928: ldr x5, [x29, #96] 0.00 : ffff80001071e92c: cmp w24, w0 0.00 : ffff80001071e930: b.eq ffff80001071e964 // b.none 0.00 : ffff80001071e934: nop 0.00 : ffff80001071e938: sxtw x0, w0 : __cmpwait_case_32(): : : [val] "r" (val)); \ : } : : __CMPWAIT_CASE(w, b, 8); : __CMPWAIT_CASE(w, h, 16); : __CMPWAIT_CASE(w, , 32); 0.00 : ffff80001071e93c: sevl 0.00 : ffff80001071e940: wfe 0.03 : ffff80001071e944: ldxr w2, [x1] 0.00 : ffff80001071e948: eor w2, w2, w0 0.00 : ffff80001071e94c: cbnz w2, ffff80001071e954 0.02 : ffff80001071e950: wfe : __read_once_size(): 0.17 : ffff80001071e954: ldr w0, [x28, #264] : arm_smmu_cmdq_issue_cmdlist(): 0.60 : ffff80001071e958: ldr w2, [x23] 0.00 : ffff80001071e95c: cmp w2, w0 0.00 : ffff80001071e960: b.ne ffff80001071e938 // b.any : arch_static_branch_jump(): 0.00 : ffff80001071e964: b ffff80001071eaa8 0.00 : ffff80001071e968: b ffff80001071eaa8 : __lse_atomic_fetch_andnot_relaxed(): : ATOMIC_FETCH_OPS(andnot, ldclr) 0.00 : ffff80001071e96c: mov w24, #0x80000000 // #-2147483648 0.00 : ffff80001071e970: add x0, x28, #0x40 0.00 : ffff80001071e974: ldclr w24, w24, [x0] 11.20 : ffff80001071e978: b ffff80001071eab4 : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e97c: msr daif, x21 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff80001071e980: nop : arch_local_save_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e984: mrs x4, daif : arch_irqs_disabled_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e988: and w0, w4, #0x80 : arch_local_irq_save(): : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff80001071e98c: cbnz w0, ffff80001071e998 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e990: mov x0, #0x60 // #96 0.00 : ffff80001071e994: msr daifset, #0x2 : atomic_cmpxchg_relaxed(): : #if defined(arch_atomic_cmpxchg_relaxed) : static inline int : atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) : { : kasan_check_write(v, sizeof(*v)); : return arch_atomic_cmpxchg_relaxed(v, old, new); 0.00 : ffff80001071e998: add x3, x28, #0x10c : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff80001071e99c: b ffff80001071ea04 0.00 : ffff80001071e9a0: b ffff80001071ea04 : __lse__cmpxchg_case_32(): : __CMPXCHG_CASE(w, , , 32, ) 0.00 : ffff80001071e9a4: mov w1, #0x0 // #0 0.00 : ffff80001071e9a8: mov x0, x3 0.00 : ffff80001071e9ac: mov w2, #0x80000000 // #-2147483648 0.00 : ffff80001071e9b0: mov w5, w1 0.00 : ffff80001071e9b4: cas w5, w2, [x3] 0.00 : ffff80001071e9b8: mov w0, w5 : arm_smmu_cmdq_poll_until_not_full(): : if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) { 0.00 : ffff80001071e9bc: cbnz w0, ffff80001071ea18 : __raw_readl(): : : #define __raw_readl __raw_readl : static __always_inline u32 __raw_readl(const volatile void __iomem *addr) : { : u32 val; : asm volatile(ALTERNATIVE("ldr %w0, [%1]", 0.00 : ffff80001071e9c0: ldr x1, [x27, #176] 0.00 : ffff80001071e9c4: ldr w1, [x1] : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001071e9c8: str w1, [x28, #68] : atomic_set_release(): : : #ifndef atomic_set_release : static inline void : atomic_set_release(atomic_t *v, int i) : { : smp_store_release(&(v)->counter, i); 0.00 : ffff80001071e9cc: add x1, x28, #0x10c 0.00 : ffff80001071e9d0: stlr w0, [x1] : arch_local_irq_restore(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e9d4: msr daif, x4 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff80001071e9d8: nop : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001071e9dc: ldr x0, [x28, #64] : arm_smmu_cmdq_poll_until_not_full(): : llq->val = READ_ONCE(cmdq->q.llq.val); 0.00 : ffff80001071e9e0: str x0, [x23] 0.00 : ffff80001071e9e4: nop : arch_local_save_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e9e8: mrs x8, daif 0.00 : ffff80001071e9ec: mov x21, x8 : arch_irqs_disabled_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e9f0: and w0, w8, #0x80 : arch_local_irq_save(): : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff80001071e9f4: cbnz w0, ffff80001071e7d0 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071e9f8: mov x0, #0x60 // #96 0.00 : ffff80001071e9fc: msr daifset, #0x2 0.00 : ffff80001071ea00: b ffff80001071e7d0 : __ll_sc__cmpxchg_case_32(): : __CMPXCHG_CASE(w, , , 32, , , , , K) 0.00 : ffff80001071ea04: mov x1, #0x0 // #0 0.00 : ffff80001071ea08: mov w2, #0x80000000 // #-2147483648 0.00 : ffff80001071ea0c: add x5, x28, #0x10c 0.00 : ffff80001071ea10: b ffff8000107217cc : arm_smmu_cmdq_poll_until_not_full(): : if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) { 0.00 : ffff80001071ea14: cbz w0, ffff80001071e9c0 : arch_local_irq_restore(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071ea18: msr daif, x4 : arch_static_branch(): 0.00 : ffff80001071ea1c: nop : queue_poll_init(): : qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); 0.00 : ffff80001071ea20: ldr w0, [x28, #16] : qp->delay = 1; 0.00 : ffff80001071ea24: mov x1, #0x1 // #1 0.00 : ffff80001071ea28: str x1, [x29, #120] : qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); 0.00 : ffff80001071ea2c: ubfx x0, x0, #6, #1 0.00 : ffff80001071ea30: strb w0, [x29, #128] : qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); 0.00 : ffff80001071ea34: bl ffff80001016c938 : ktime_add_us(): : return ktime_to_ms(ktime_sub(later, earlier)); : } : : static inline ktime_t ktime_add_us(const ktime_t kt, const u64 usec) : { : return ktime_add_ns(kt, usec * NSEC_PER_USEC); 0.00 : ffff80001071ea38: mov x1, #0xca00 // #51712 0.00 : ffff80001071ea3c: movk x1, #0x3b9a, lsl #16 0.00 : ffff80001071ea40: add x0, x0, x1 : queue_poll_init(): 0.00 : ffff80001071ea44: str x0, [x29, #112] : __read_once_size(): 0.00 : ffff80001071ea48: ldr x0, [x28, #64] : arm_smmu_cmdq_poll_until_not_full(): : llq->val = READ_ONCE(smmu->cmdq.q.llq.val); 0.00 : ffff80001071ea4c: str x0, [x23] : queue_full(): : return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && 0.00 : ffff80001071ea50: lsr x1, x0, #32 0.00 : ffff80001071ea54: eor w0, w0, w1 0.00 : ffff80001071ea58: tst w22, w0 0.00 : ffff80001071ea5c: b.ne ffff80001071e9e8 // b.any 0.00 : ffff80001071ea60: tst w19, w0 0.00 : ffff80001071ea64: b.eq ffff80001071e9e8 // b.none : arm_smmu_cmdq_poll_until_not_full(): : ret = queue_poll(&qp); 0.00 : ffff80001071ea68: add x0, x29, #0x70 0.00 : ffff80001071ea6c: bl ffff80001071e2e8 : } while (!ret); 0.00 : ffff80001071ea70: cbz w0, ffff80001071ea48 : arm_smmu_cmdq_issue_cmdlist(): : dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); 0.00 : ffff80001071ea74: adrp x0, ffff800011a88000 0.00 : ffff80001071ea78: add x0, x0, #0xae8 0.00 : ffff80001071ea7c: mov x1, x24 0.00 : ffff80001071ea80: add x0, x0, #0x138 0.00 : ffff80001071ea84: bl ffff800010cc4388 <___ratelimit> 0.00 : ffff80001071ea88: cbz w0, ffff80001071e9e8 0.00 : ffff80001071ea8c: ldr x0, [x28] 0.00 : ffff80001071ea90: adrp x1, ffff80001126a000 0.00 : ffff80001071ea94: add x1, x1, #0x888 0.00 : ffff80001071ea98: bl ffff80001072f050 <_dev_err> 0.00 : ffff80001071ea9c: b ffff80001071e9e8 : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff80001071eaa0: dsb sy 0.00 : ffff80001071eaa4: b ffff80001071e984 : __ll_sc_atomic_fetch_andnot_relaxed(): : ATOMIC_OPS(andnot, bic, ) 0.00 : ffff80001071eaa8: mov w0, #0x80000000 // #-2147483648 0.00 : ffff80001071eaac: add x3, x28, #0x40 0.00 : ffff80001071eab0: b ffff8000107217e8 : arm_smmu_cmdq_poll_valid_map(): : __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false); 0.00 : ffff80001071eab4: ldr w0, [x28, #128] : arm_smmu_cmdq_issue_cmdlist(): : : /* b. Stop gathering work by clearing the owned flag */ : prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG, : &cmdq->q.llq.atomic.prod); : prod &= ~CMDQ_PROD_OWNED_FLAG; 0.00 : ffff80001071eab8: and w24, w24, #0x7fffffff : arm_smmu_cmdq_poll_valid_map(): : __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false); 0.00 : ffff80001071eabc: ldr w2, [x26, #128] 0.00 : ffff80001071eac0: mov w4, #0x0 // #0 0.00 : ffff80001071eac4: mov w3, w24 0.00 : ffff80001071eac8: mov x1, x5 0.00 : ffff80001071eacc: bl ffff80001071e4e8 <__arm_smmu_cmdq_poll_set_valid_map.isra.33> : __raw_writel(): : asm volatile("str %w0, [%1]" : : "rZ" (val), "r" (addr)); 0.05 : ffff80001071ead0: ldr x0, [x27, #168] 0.00 : ffff80001071ead4: str w24, [x0] : atomic_set_release(): 0.00 : ffff80001071ead8: add x0, x28, #0x108 0.03 : ffff80001071eadc: stlr w24, [x0] : arm_smmu_cmdq_issue_cmdlist(): : */ : atomic_set_release(&cmdq->owner_prod, prod); : } : : /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */ : if (sync) { 3.14 : ffff80001071eae0: ldr w0, [x29, #108] : int ret = 0; 0.00 : ffff80001071eae4: mov w24, #0x0 // #0 : if (sync) { 0.00 : ffff80001071eae8: cbnz w0, ffff80001071eb44 : arch_local_irq_restore(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001071eaec: msr daif, x21 : arch_static_branch(): 0.25 : ffff80001071eaf0: nop : arm_smmu_cmdq_issue_cmdlist(): : } : } : : local_irq_restore(flags); : return ret; : } 0.00 : ffff80001071eaf4: adrp x0, ffff800011909000 0.00 : ffff80001071eaf8: add x21, x0, #0x908 0.00 : ffff80001071eafc: ldr x2, [x29, #488] 0.22 : ffff80001071eb00: ldr x1, [x21] 0.00 : ffff80001071eb04: eor x1, x2, x1 0.00 : ffff80001071eb08: mov w0, w24 0.00 : ffff80001071eb0c: cbnz x1, ffff80001071ee8c 0.00 : ffff80001071eb10: ldp x19, x20, [sp, #16] 0.00 : ffff80001071eb14: ldp x21, x22, [sp, #32] 0.00 : ffff80001071eb18: ldp x23, x24, [sp, #48] 0.00 : ffff80001071eb1c: ldp x25, x26, [sp, #64] 0.00 : ffff80001071eb20: ldp x27, x28, [sp, #80] 0.00 : ffff80001071eb24: ldp x29, x30, [sp], #496 0.00 : ffff80001071eb28: ret : arch_local_irq_restore(): : pmr_sync(); 0.00 : ffff80001071eb2c: dsb sy 0.00 : ffff80001071eb30: b ffff80001071e9dc 0.00 : ffff80001071eb34: dsb sy 0.00 : ffff80001071eb38: b ffff80001071ea20 0.01 : ffff80001071eb3c: dsb sy : arm_smmu_cmdq_issue_cmdlist(): : return ret; 1.22 : ffff80001071eb40: b ffff80001071eaf4 : llq.prod = queue_inc_prod_n(&llq, n); 0.00 : ffff80001071eb44: ldr w2, [x26, #128] : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071eb48: ldr w3, [x29, #104] 0.00 : ffff80001071eb4c: and w0, w2, w20 : arm_smmu_cmdq_poll_until_sync(): : if (smmu->features & ARM_SMMU_FEAT_MSI && 0.05 : ffff80001071eb50: ldr w1, [x28, #16] : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071eb54: add w3, w0, w3 : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.00 : ffff80001071eb58: and w2, w2, #0x80000000 0.00 : ffff80001071eb5c: and w3, w3, w20 : arm_smmu_cmdq_poll_until_sync(): : if (smmu->features & ARM_SMMU_FEAT_MSI && 3.09 : ffff80001071eb60: and w0, w1, #0x180 : queue_inc_prod_n(): : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.00 : ffff80001071eb64: orr w25, w3, w2 : arm_smmu_cmdq_issue_cmdlist(): : llq.prod = queue_inc_prod_n(&llq, n); 0.00 : ffff80001071eb68: str w25, [x26, #128] 0.00 : ffff80001071eb6c: ubfx x1, x1, #6, #1 : arm_smmu_cmdq_poll_until_sync(): : if (smmu->features & ARM_SMMU_FEAT_MSI && 0.00 : ffff80001071eb70: cmp w0, #0x180 0.00 : ffff80001071eb74: b.eq ffff80001071edac // b.none : queue_poll_init(): : qp->delay = 1; 0.00 : ffff80001071eb78: mov x0, #0x1 // #1 0.00 : ffff80001071eb7c: str x0, [x29, #120] : qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); 0.00 : ffff80001071eb80: strb w1, [x29, #128] 0.00 : ffff80001071eb84: and w20, w22, w25 : qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); 0.00 : ffff80001071eb88: bl ffff80001016c938 : __read_once_size(): 0.00 : ffff80001071eb8c: ldr x1, [x28, #64] : ktime_add_us(): 0.00 : ffff80001071eb90: mov x2, #0xca00 // #51712 : __arm_smmu_cmdq_poll_until_consumed(): : llq->val = READ_ONCE(smmu->cmdq.q.llq.val); 0.00 : ffff80001071eb94: str x1, [x26, #128] : ktime_add_us(): 0.00 : ffff80001071eb98: movk x2, #0x3b9a, lsl #16 0.00 : ffff80001071eb9c: add x0, x0, x2 : queue_poll_init(): : qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); 0.00 : ffff80001071eba0: str x0, [x29, #112] 0.00 : ffff80001071eba4: lsr x1, x1, #32 : queue_consumed(): : return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) && 0.00 : ffff80001071eba8: eor w0, w25, w1 0.00 : ffff80001071ebac: and w1, w22, w1 : (Q_IDX(q, q->cons) > Q_IDX(q, prod))) || 0.00 : ffff80001071ebb0: tst w0, w19 0.00 : ffff80001071ebb4: b.eq ffff80001071ebf4 // b.none : ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) && 0.00 : ffff80001071ebb8: cmp w20, w1 0.00 : ffff80001071ebbc: b.cc ffff80001071ebfc // b.lo, b.ul, b.last : __arm_smmu_cmdq_poll_until_consumed(): 0.00 : ffff80001071ebc0: mov w24, #0x0 // #0 : __read_once_size(): 0.03 : ffff80001071ebc4: ldr w1, [x28, #268] 0.00 : ffff80001071ebc8: add x0, x28, #0x10c : arm_smmu_cmdq_shared_tryunlock(): : if (atomic_read(&cmdq->lock) == 1) 0.00 : ffff80001071ebcc: cmp w1, #0x1 2.61 : ffff80001071ebd0: b.eq ffff80001071ee44 // b.none : arch_static_branch_jump(): : asm_volatile_goto( 0.29 : ffff80001071ebd4: b ffff80001071ec70 0.01 : ffff80001071ebd8: b ffff80001071ec70 : __lse_atomic_sub_return_release(): : ATOMIC_OP_SUB_RETURN(_release, l, "memory") 0.00 : ffff80001071ebdc: mov w1, #0x1 // #1 0.00 : ffff80001071ebe0: add x3, x28, #0x10c 0.00 : ffff80001071ebe4: neg w1, w1 0.00 : ffff80001071ebe8: ldaddl w1, w2, [x3] 6.52 : ffff80001071ebec: add w1, w1, w2 0.00 : ffff80001071ebf0: b ffff80001071eaec : queue_consumed(): : return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) && 0.00 : ffff80001071ebf4: cmp w20, w1 0.00 : ffff80001071ebf8: b.cc ffff80001071ebc0 // b.lo, b.ul, b.last : __arm_smmu_cmdq_poll_until_consumed(): : ret = queue_poll(&qp); 0.00 : ffff80001071ebfc: add x0, x29, #0x70 0.00 : ffff80001071ec00: bl ffff80001071e2e8 : __raw_readl(): : asm volatile(ALTERNATIVE("ldr %w0, [%1]", 0.00 : ffff80001071ec04: ldr x1, [x27, #176] : __arm_smmu_cmdq_poll_until_consumed(): 0.00 : ffff80001071ec08: mov w24, w0 : __raw_readl(): 0.00 : ffff80001071ec0c: ldr w1, [x1] : __arm_smmu_cmdq_poll_until_consumed(): : llq->cons = readl(cmdq->q.cons_reg); 0.00 : ffff80001071ec10: dmb oshld 0.00 : ffff80001071ec14: mov w0, w1 0.00 : ffff80001071ec18: eor x0, x0, x0 0.00 : ffff80001071ec1c: cbnz x0, ffff80001071ec1c 0.00 : ffff80001071ec20: str w1, [x23, #4] : } while (!ret); 0.00 : ffff80001071ec24: cbz w24, ffff80001071eba8 : arm_smmu_cmdq_issue_cmdlist(): : dev_err_ratelimited(smmu->dev, 0.00 : ffff80001071ec28: adrp x1, ffff800010ed9000 0.00 : ffff80001071ec2c: adrp x0, ffff800011a88000 0.00 : ffff80001071ec30: add x1, x1, #0x458 0.00 : ffff80001071ec34: add x0, x0, #0xae8 0.00 : ffff80001071ec38: add x1, x1, #0x20 0.00 : ffff80001071ec3c: add x0, x0, #0x110 0.00 : ffff80001071ec40: bl ffff800010cc4388 <___ratelimit> 0.00 : ffff80001071ec44: cbz w0, ffff80001071ebc4 : __raw_readl(): 0.00 : ffff80001071ec48: ldr x3, [x27, #168] 0.00 : ffff80001071ec4c: ldr w3, [x3] 0.00 : ffff80001071ec50: ldr x4, [x27, #176] 0.00 : ffff80001071ec54: ldr w4, [x4] : arm_smmu_cmdq_issue_cmdlist(): 0.00 : ffff80001071ec58: ldr w2, [x26, #128] 0.00 : ffff80001071ec5c: adrp x1, ffff80001126a000 0.00 : ffff80001071ec60: ldr x0, [x28] 0.00 : ffff80001071ec64: add x1, x1, #0x898 0.00 : ffff80001071ec68: bl ffff80001072f050 <_dev_err> 0.00 : ffff80001071ec6c: b ffff80001071ebc4 : __ll_sc_atomic_sub_return_release(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff80001071ec70: mov w0, #0x1 // #1 0.00 : ffff80001071ec74: add x3, x28, #0x10c 0.00 : ffff80001071ec78: b ffff800010721800 0.00 : ffff80001071ec7c: b ffff80001071eaec : arm_smmu_cmdq_build_sync_cmd(): : struct arm_smmu_cmdq_ent ent = { 0.01 : ffff80001071ec80: mov w1, #0x46 // #70 0.00 : ffff80001071ec84: stp xzr, xzr, [x29, #112] 0.00 : ffff80001071ec88: strb w1, [x29, #112] : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071ec8c: and w9, w12, w20 : arm_smmu_cmdq_build_sync_cmd(): : if (smmu->features & ARM_SMMU_FEAT_MSI && 0.00 : ffff80001071ec90: ldr w0, [x28, #16] : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 1.26 : ffff80001071ec94: ldr w1, [x29, #104] : arm_smmu_cmdq_build_sync_cmd(): : struct arm_smmu_cmdq_ent ent = { 0.00 : ffff80001071ec98: stp xzr, xzr, [x29, #128] : if (smmu->features & ARM_SMMU_FEAT_MSI && 0.00 : ffff80001071ec9c: and w0, w0, #0x180 : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071eca0: add w9, w9, w1 : arm_smmu_cmdq_build_sync_cmd(): : if (smmu->features & ARM_SMMU_FEAT_MSI && 0.00 : ffff80001071eca4: cmp w0, #0x180 : queue_inc_prod_n(): : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.00 : ffff80001071eca8: and w9, w9, w20 : arm_smmu_cmdq_build_sync_cmd(): : if (smmu->features & ARM_SMMU_FEAT_MSI && 0.00 : ffff80001071ecac: b.ne ffff80001071ecd8 // b.any : ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) * 0.00 : ffff80001071ecb0: ldr w3, [x27, #64] 0.00 : ffff80001071ecb4: mov w0, #0x1 // #1 : q->ent_dwords * 8; 0.01 : ffff80001071ecb8: ldr x1, [x27, #160] : ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) * 0.00 : ffff80001071ecbc: ldr x2, [x27, #144] 0.00 : ffff80001071ecc0: lsl w0, w0, w3 0.00 : ffff80001071ecc4: sub w0, w0, #0x1 0.00 : ffff80001071ecc8: and w0, w0, w9 : q->ent_dwords * 8; 0.00 : ffff80001071eccc: lsl x1, x1, #3 : ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) * 0.00 : ffff80001071ecd0: madd x0, x0, x1, x2 0.00 : ffff80001071ecd4: str x0, [x29, #120] : arm_smmu_cmdq_build_cmd(cmd, &ent); 0.00 : ffff80001071ecd8: add x1, x29, #0x70 0.00 : ffff80001071ecdc: add x0, x29, #0x1d8 0.00 : ffff80001071ece0: bl ffff80001071d5e0 : arm_smmu_cmdq_issue_cmdlist(): : queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS); 0.00 : ffff80001071ece4: ldr w4, [x27, #64] 0.00 : ffff80001071ece8: mov w0, #0x1 // #1 0.00 : ffff80001071ecec: ldr x1, [x27, #160] 0.01 : ffff80001071ecf0: ldr x2, [x27, #136] 0.00 : ffff80001071ecf4: lsl w0, w0, w4 0.00 : ffff80001071ecf8: sub w0, w0, #0x1 0.00 : ffff80001071ecfc: lsl x1, x1, #3 0.00 : ffff80001071ed00: and w0, w0, w9 : queue_write(): : *dst++ = cpu_to_le64(*src++); 0.05 : ffff80001071ed04: ldr x3, [x29, #472] : arm_smmu_cmdq_shared_lock(): : if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0) 0.00 : ffff80001071ed08: add x4, x28, #0x10c : arm_smmu_cmdq_issue_cmdlist(): : queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS); 0.00 : ffff80001071ed0c: mul x0, x0, x1 0.00 : ffff80001071ed10: add x1, x2, x0 : queue_write(): : *dst++ = cpu_to_le64(*src++); 0.05 : ffff80001071ed14: str x3, [x2, x0] 0.02 : ffff80001071ed18: ldr x0, [x29, #480] 0.07 : ffff80001071ed1c: str x0, [x1, #8] : arch_static_branch_jump(): 0.00 : ffff80001071ed20: b ffff80001071ed94 0.00 : ffff80001071ed24: b ffff80001071ed94 : __lse_atomic_fetch_add_relaxed(): : ATOMIC_FETCH_OPS(add, ldadd) 0.00 : ffff80001071ed28: mov w0, #0x1 // #1 0.00 : ffff80001071ed2c: ldadd w0, w0, [x4] : arm_smmu_cmdq_shared_lock(): : if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0) 16.85 : ffff80001071ed30: tbz w0, #31, ffff80001071e8f4 : __read_once_size(): 0.00 : ffff80001071ed34: ldr w3, [x28, #268] : arm_smmu_cmdq_shared_lock(): : val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0); 0.00 : ffff80001071ed38: tbz w3, #31, ffff80001071ed64 0.00 : ffff80001071ed3c: nop 0.00 : ffff80001071ed40: sxtw x3, w3 : __cmpwait_case_32(): 0.00 : ffff80001071ed44: sevl 0.00 : ffff80001071ed48: wfe 0.00 : ffff80001071ed4c: ldxr w0, [x4] 0.00 : ffff80001071ed50: eor w0, w0, w3 0.00 : ffff80001071ed54: cbnz w0, ffff80001071ed5c 0.00 : ffff80001071ed58: wfe : __read_once_size(): 0.00 : ffff80001071ed5c: ldr w3, [x28, #268] : arm_smmu_cmdq_shared_lock(): 0.00 : ffff80001071ed60: tbnz w3, #31, ffff80001071ed40 : atomic_cmpxchg_relaxed(): 0.00 : ffff80001071ed64: sxtw x1, w3 : arm_smmu_cmdq_shared_lock(): : } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val); 0.00 : ffff80001071ed68: add w2, w3, #0x1 : arch_static_branch_jump(): 0.00 : ffff80001071ed6c: b ffff80001071eda4 0.00 : ffff80001071ed70: b ffff80001071eda4 : __lse__cmpxchg_case_32(): : __CMPXCHG_CASE(w, , , 32, ) 0.00 : ffff80001071ed74: mov x0, x4 0.00 : ffff80001071ed78: mov w1, w3 0.00 : ffff80001071ed7c: mov w5, w1 0.00 : ffff80001071ed80: cas w5, w2, [x4] 0.00 : ffff80001071ed84: mov w0, w5 : arm_smmu_cmdq_shared_lock(): 0.00 : ffff80001071ed88: cmp w3, w0 0.00 : ffff80001071ed8c: b.ne ffff80001071ed5c // b.any 0.00 : ffff80001071ed90: b ffff80001071e8f4 : __ll_sc_atomic_fetch_add_relaxed(): : ATOMIC_OPS(add, add, I) 0.00 : ffff80001071ed94: add x3, x28, #0x10c 0.00 : ffff80001071ed98: b ffff800010721818 : arm_smmu_cmdq_shared_lock(): : if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0) 0.00 : ffff80001071ed9c: tbz w0, #31, ffff80001071e8f4 0.00 : ffff80001071eda0: b ffff80001071ed34 : __ll_sc__cmpxchg_case_32(): : __CMPXCHG_CASE(w, , , 32, , , , , K) 0.00 : ffff80001071eda4: b ffff800010721830 0.00 : ffff80001071eda8: b ffff80001071ed88 : __arm_smmu_cmdq_poll_until_msi(): : u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod)); 0.00 : ffff80001071edac: ldr w2, [x27, #64] 0.00 : ffff80001071edb0: mov w19, #0x1 // #1 0.01 : ffff80001071edb4: ldr x0, [x27, #160] : queue_poll_init(): : qp->delay = 1; 0.00 : ffff80001071edb8: mov x3, #0x1 // #1 : __arm_smmu_cmdq_poll_until_msi(): : u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod)); 0.00 : ffff80001071edbc: ldr x24, [x27, #136] 0.00 : ffff80001071edc0: lsl w19, w19, w2 0.00 : ffff80001071edc4: sub w19, w19, #0x1 0.00 : ffff80001071edc8: lsl x0, x0, #3 0.00 : ffff80001071edcc: and w19, w19, w25 : queue_poll_init(): : qp->delay = 1; 0.00 : ffff80001071edd0: str x3, [x29, #120] : qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); 0.01 : ffff80001071edd4: strb w1, [x29, #128] : __arm_smmu_cmdq_poll_until_msi(): : u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod)); 0.00 : ffff80001071edd8: mul x19, x19, x0 : queue_poll_init(): : qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); 0.00 : ffff80001071eddc: bl ffff80001016c938 : ktime_add_us(): 0.00 : ffff80001071ede0: mov x1, #0xca00 // #51712 : __arm_smmu_cmdq_poll_until_msi(): : qp.wfe = false; 0.00 : ffff80001071ede4: strb wzr, [x29, #128] : ktime_add_us(): 0.00 : ffff80001071ede8: movk x1, #0x3b9a, lsl #16 0.00 : ffff80001071edec: add x0, x0, x1 : queue_poll_init(): : qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); 0.01 : ffff80001071edf0: str x0, [x29, #112] : __arm_smmu_cmdq_poll_until_msi(): : u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod)); 0.00 : ffff80001071edf4: add x22, x24, x19 : __read_once_size(): 0.00 : ffff80001071edf8: ldr w19, [x24, x19] : __arm_smmu_cmdq_poll_until_msi(): : smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp))); 0.00 : ffff80001071edfc: cbnz w19, ffff80001071ee28 0.55 : ffff80001071ee00: b ffff80001071ee6c 0.00 : ffff80001071ee04: mov w19, w19 : __cmpwait_case_32(): 0.01 : ffff80001071ee08: sevl 0.00 : ffff80001071ee0c: wfe 0.61 : ffff80001071ee10: ldxr w0, [x22] 0.00 : ffff80001071ee14: eor w0, w0, w19 0.00 : ffff80001071ee18: cbnz w0, ffff80001071ee20 0.35 : ffff80001071ee1c: wfe : __read_once_size(): 2.51 : ffff80001071ee20: ldr w19, [x22] : __arm_smmu_cmdq_poll_until_msi(): 0.00 : ffff80001071ee24: cbz w19, ffff80001071ee68 1.90 : ffff80001071ee28: add x0, x29, #0x70 0.00 : ffff80001071ee2c: bl ffff80001071e2e8 0.00 : ffff80001071ee30: mov w24, w0 0.00 : ffff80001071ee34: cbz w0, ffff80001071ee04 : llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1); 0.00 : ffff80001071ee38: ldr w0, [x26, #128] 0.00 : ffff80001071ee3c: str w0, [x23, #4] 0.00 : ffff80001071ee40: b ffff80001071ec28 : arm_smmu_cmdq_issue_cmdlist(): : WRITE_ONCE(cmdq->q.llq.cons, llq.cons); 0.06 : ffff80001071ee44: ldr w1, [x23, #4] : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.03 : ffff80001071ee48: str w1, [x28, #68] : arch_static_branch_jump(): 0.00 : ffff80001071ee4c: b ffff80001071ec70 0.00 : ffff80001071ee50: b ffff80001071ec70 : __lse_atomic_sub_return_release(): : ATOMIC_OP_SUB_RETURN(_release, l, "memory") 0.00 : ffff80001071ee54: mov w1, #0x1 // #1 0.00 : ffff80001071ee58: neg w1, w1 0.00 : ffff80001071ee5c: ldaddl w1, w2, [x0] 6.65 : ffff80001071ee60: add w1, w1, w2 0.00 : ffff80001071ee64: b ffff80001071eaec 1.79 : ffff80001071ee68: ldr w25, [x26, #128] : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071ee6c: and w0, w20, w25 : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.00 : ffff80001071ee70: and w22, w25, #0x80000000 : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071ee74: add w0, w0, #0x1 : __arm_smmu_cmdq_poll_until_msi(): 0.14 : ffff80001071ee78: mov w24, #0x0 // #0 : queue_inc_prod_n(): : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.05 : ffff80001071ee7c: and w20, w0, w20 0.00 : ffff80001071ee80: orr w22, w20, w22 : __arm_smmu_cmdq_poll_until_msi(): : llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1); 0.00 : ffff80001071ee84: str w22, [x23, #4] 0.00 : ffff80001071ee88: b ffff80001071ebc4 : arm_smmu_cmdq_issue_cmdlist(): : } 0.00 : ffff80001071ee8c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (207096 samples, percent: local period) ------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd8540 <_raw_spin_unlock_irqrestore>: : _raw_spin_unlock_irqrestore(): : EXPORT_SYMBOL(_raw_spin_unlock); : #endif : : #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE : void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) : { 0.89 : ffff800010cd8540: stp x29, x30, [sp, #-16]! : queued_spin_unlock(): : static __always_inline void queued_spin_unlock(struct qspinlock *lock) : { : /* : * unlock() needs release semantics: : */ : smp_store_release(&lock->locked, 0); 0.00 : ffff800010cd8544: mov w2, #0x0 // #0 : _raw_spin_unlock_irqrestore(): 0.00 : ffff800010cd8548: mov x29, sp : queued_spin_unlock(): 0.10 : ffff800010cd854c: stlrb w2, [x0] : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 14.74 : ffff800010cd8550: msr daif, x1 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 16.27 : ffff800010cd8554: nop : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010cd8558: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010cd855c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.01 : ffff800010cd8560: sub x0, x0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 11.26 : ffff800010cd8564: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010cd8568: cbz x0, ffff800010cd8574 <_raw_spin_unlock_irqrestore+0x34> : __read_once_size(): : __READ_ONCE_SIZE; 0.02 : ffff800010cd856c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010cd8570: cbnz x0, ffff800010cd8578 <_raw_spin_unlock_irqrestore+0x38> : __raw_spin_unlock_irqrestore(): : unsigned long flags) : { : spin_release(&lock->dep_map, _RET_IP_); : do_raw_spin_unlock(lock); : local_irq_restore(flags); : preempt_enable(); 0.00 : ffff800010cd8574: bl ffff800010cd2e50 : _raw_spin_unlock_irqrestore(): : __raw_spin_unlock_irqrestore(lock, flags); : } 1.30 : ffff800010cd8578: ldp x29, x30, [sp], #16 0.00 : ffff800010cd857c: ret : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.71 : ffff800010cd8580: dsb sy 54.70 : ffff800010cd8584: b ffff800010cd8558 <_raw_spin_unlock_irqrestore+0x18> Percent | Source code & Disassembly of vmlinux for cycles (198567 samples, percent: local period) ------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016c938 : : ktime_get(): : timespec64_add_ns(ts, nsecs); : } : EXPORT_SYMBOL(ktime_get_real_ts64); : : ktime_t ktime_get(void) : { 0.59 : ffff80001016c938: stp x29, x30, [sp, #-48]! : struct timekeeper *tk = &tk_core.timekeeper; : unsigned int seq; : ktime_t base; : u64 nsecs; : : WARN_ON(timekeeping_suspended); 0.00 : ffff80001016c93c: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> : { 0.00 : ffff80001016c940: mov x29, sp : WARN_ON(timekeeping_suspended); 0.01 : ffff80001016c944: ldr w0, [x0, #708] : { 0.18 : ffff80001016c948: stp x19, x20, [sp, #16] 0.21 : ffff80001016c94c: stp x21, x22, [sp, #32] : WARN_ON(timekeeping_suspended); 0.00 : ffff80001016c950: cbnz w0, ffff80001016c9c8 0.20 : ffff80001016c954: adrp x20, ffff800011b20000 <__log_buf+0x1fc10> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001016c958: add x20, x20, #0xec0 0.01 : ffff80001016c95c: ldr w22, [x20, #64] : __read_seqcount_begin(): : { : unsigned ret; : : repeat: : ret = READ_ONCE(s->sequence); : if (unlikely(ret & 1)) { 0.00 : ffff80001016c960: tbnz w22, #0, ffff80001016c9c0 : raw_read_seqcount_begin(): : * section is tested by checking read_seqcount_retry function. : */ : static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) : { : unsigned ret = __read_seqcount_begin(s); : smp_rmb(); 0.53 : ffff80001016c964: dmb ishld : __read_once_size(): 92.07 : ffff80001016c968: ldr x1, [x20, #72] : ktime_get(): : : do { : seq = read_seqcount_begin(&tk_core.seq); : base = tk->tkr_mono.base; 0.01 : ffff80001016c96c: ldr x21, [x20, #112] : tk_clock_read(): : return clock->read(clock); 0.00 : ffff80001016c970: mov x0, x1 0.00 : ffff80001016c974: ldr x1, [x1] 0.00 : ffff80001016c978: blr x1 : timekeeping_get_ns(): : return timekeeping_delta_to_ns(tkr, delta); 0.00 : ffff80001016c97c: ldp w1, w3, [x20, #96] : timekeeping_get_delta(): : delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask); 0.01 : ffff80001016c980: ldp x5, x6, [x20, #80] : timekeeping_get_ns(): : return timekeeping_delta_to_ns(tkr, delta); 0.00 : ffff80001016c984: ldr x4, [x20, #104] : read_seqcount_retry(): : * If the critical section was invalid, it must be ignored (and typically : * retried). : */ : static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) : { : smp_rmb(); 0.53 : ffff80001016c988: dmb ishld : ktime_get(): : nsecs = timekeeping_get_ns(&tk->tkr_mono); : : } while (read_seqcount_retry(&tk_core.seq, seq)); 4.88 : ffff80001016c98c: ldr w2, [x20, #64] 0.00 : ffff80001016c990: cmp w22, w2 0.00 : ffff80001016c994: b.ne ffff80001016c95c // b.any : clocksource_delta(): : return ret & ~(mask >> 1) ? 0 : ret; : } : #else : static inline u64 clocksource_delta(u64 now, u64 last, u64 mask) : { : return (now - last) & mask; 0.00 : ffff80001016c998: sub x0, x0, x6 : timekeeping_delta_to_ns(): : nsec = delta * tkr->mult + tkr->xtime_nsec; 0.00 : ffff80001016c99c: mov w1, w1 : clocksource_delta(): 0.00 : ffff80001016c9a0: and x0, x0, x5 : ktime_get(): : : return ktime_add_ns(base, nsecs); : } 0.00 : ffff80001016c9a4: ldp x19, x20, [sp, #16] : timekeeping_delta_to_ns(): : nsec = delta * tkr->mult + tkr->xtime_nsec; 0.00 : ffff80001016c9a8: madd x0, x0, x1, x4 : nsec >>= tkr->shift; 0.00 : ffff80001016c9ac: lsr x0, x0, x3 : ktime_get(): : } 0.00 : ffff80001016c9b0: add x0, x0, x21 0.45 : ffff80001016c9b4: ldp x21, x22, [sp, #32] 0.06 : ffff80001016c9b8: ldp x29, x30, [sp], #48 0.00 : ffff80001016c9bc: ret : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.20 : ffff80001016c9c0: yield 0.06 : ffff80001016c9c4: b ffff80001016c95c : ktime_get(): : WARN_ON(timekeeping_suspended); 0.00 : ffff80001016c9c8: brk #0x800 0.00 : ffff80001016c9cc: b ffff80001016c954 Percent | Source code & Disassembly of vmlinux for cycles (191397 samples, percent: local period) ------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001071f1c8 : : arm_smmu_atc_inv_domain(): : } : : return arm_smmu_cmdq_issue_sync(master->smmu); : } : : static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, 0.02 : ffff80001071f1c8: stp x29, x30, [sp, #-112]! 0.00 : ffff80001071f1cc: mov x29, sp 0.03 : ffff80001071f1d0: str x20, [sp, #24] 0.00 : ffff80001071f1d4: adrp x20, ffff800011909000 0.00 : ffff80001071f1d8: add x3, x20, #0x908 0.05 : ffff80001071f1dc: ldr x4, [x3] 0.06 : ffff80001071f1e0: str x4, [x29, #104] 0.00 : ffff80001071f1e4: mov x4, #0x0 // #0 : int ret = 0; : unsigned long flags; : struct arm_smmu_cmdq_ent cmd; : struct arm_smmu_master *master; : : if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS)) 0.01 : ffff80001071f1e8: ldr x3, [x0] 0.03 : ffff80001071f1ec: ldr w3, [x3, #16] 0.00 : ffff80001071f1f0: tbz w3, #5, ffff80001071f2c8 : * atomic_read(&nr_ats_masters); pci_enable_ats() // writel() : * : * Ensures that we always see the incremented 'nr_ats_masters' count if : * ATS was enabled at the PCI device before completion of the TLBI. : */ : smp_mb(); 96.80 : ffff80001071f1f4: dmb ish : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.48 : ffff80001071f1f8: ldr w3, [x0, #52] : arm_smmu_atc_inv_domain(): : if (!atomic_read(&smmu_domain->nr_ats_masters)) 0.00 : ffff80001071f1fc: cbz w3, ffff80001071f2c8 : arm_smmu_atc_inv_to_cmd(): : *cmd = (struct arm_smmu_cmdq_ent) { 0.00 : ffff80001071f200: mov w3, #0x40 // #64 0.00 : ffff80001071f204: strb w3, [x29, #72] 0.00 : ffff80001071f208: add x3, x29, #0x4a 0.00 : ffff80001071f20c: str x19, [x29, #16] 0.00 : ffff80001071f210: stp x21, x22, [x29, #32] 0.00 : ffff80001071f214: str x24, [x29, #56] 0.00 : ffff80001071f218: stp xzr, xzr, [x3] 0.00 : ffff80001071f21c: strb wzr, [x29, #73] 0.00 : ffff80001071f220: stur xzr, [x29, #90] 0.00 : ffff80001071f224: stur wzr, [x29, #98] 0.00 : ffff80001071f228: strh wzr, [x29, #102] : if (!size) { 0.00 : ffff80001071f22c: cbnz x2, ffff80001071f2ec : cmd->atc.size = ATC_INV_SIZE_ALL; 0.00 : ffff80001071f230: mov w1, #0x34 // #52 0.00 : ffff80001071f234: strb w1, [x29, #96] : arm_smmu_atc_inv_domain(): : return 0; : : arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd); : : spin_lock_irqsave(&smmu_domain->devices_lock, flags); : list_for_each_entry(master, &smmu_domain->devices, domain_head) 0.00 : ffff80001071f238: mov x22, x0 : spinlock_check(): : * Map the spin_lock functions to the raw variants for PREEMPT_RT=n : */ : : static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock) : { : return &lock->rlock; 0.00 : ffff80001071f23c: add x21, x0, #0xe0 : arm_smmu_atc_inv_domain(): : spin_lock_irqsave(&smmu_domain->devices_lock, flags); 0.00 : ffff80001071f240: mov x0, x21 0.00 : ffff80001071f244: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff80001071f248: mov x24, x0 : list_for_each_entry(master, &smmu_domain->devices, domain_head) 0.00 : ffff80001071f24c: ldr x0, [x22, #208]! 0.00 : ffff80001071f250: sub x19, x0, #0x18 0.00 : ffff80001071f254: cmp x22, x0 0.00 : ffff80001071f258: b.eq ffff80001071f33c // b.none 0.00 : ffff80001071f25c: str x23, [x29, #48] : int ret = 0; 0.00 : ffff80001071f260: mov w23, #0x0 // #0 0.00 : ffff80001071f264: b ffff80001071f278 : list_for_each_entry(master, &smmu_domain->devices, domain_head) 0.00 : ffff80001071f268: ldr x1, [x19, #24] 0.00 : ffff80001071f26c: sub x19, x1, #0x18 0.00 : ffff80001071f270: cmp x22, x1 0.00 : ffff80001071f274: b.eq ffff80001071f2a0 // b.none : arm_smmu_atc_inv_master(): : if (!master->ats_enabled) 0.00 : ffff80001071f278: ldrb w0, [x19, #52] 0.00 : ffff80001071f27c: cbz w0, ffff80001071f268 0.00 : ffff80001071f280: add x1, x29, #0x48 0.00 : ffff80001071f284: mov x0, x19 0.00 : ffff80001071f288: bl ffff80001071f150 0.00 : ffff80001071f28c: orr w23, w23, w0 : arm_smmu_atc_inv_domain(): : list_for_each_entry(master, &smmu_domain->devices, domain_head) 0.00 : ffff80001071f290: ldr x1, [x19, #24] 0.00 : ffff80001071f294: sub x19, x1, #0x18 0.00 : ffff80001071f298: cmp x22, x1 0.00 : ffff80001071f29c: b.ne ffff80001071f278 // b.any : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff80001071f2a0: mov x1, x24 0.00 : ffff80001071f2a4: mov x0, x21 0.00 : ffff80001071f2a8: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : arm_smmu_atc_inv_domain(): : ret |= arm_smmu_atc_inv_master(master, &cmd); : spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); : : return ret ? -ETIMEDOUT : 0; 0.00 : ffff80001071f2ac: cbnz w23, ffff80001071f34c 0.00 : ffff80001071f2b0: ldr x23, [x29, #48] 0.00 : ffff80001071f2b4: mov w0, #0x0 // #0 0.00 : ffff80001071f2b8: ldr x19, [x29, #16] 0.00 : ffff80001071f2bc: ldp x21, x22, [x29, #32] 0.00 : ffff80001071f2c0: ldr x24, [x29, #56] 0.00 : ffff80001071f2c4: b ffff80001071f2cc : return 0; 0.30 : ffff80001071f2c8: mov w0, #0x0 // #0 : } 0.00 : ffff80001071f2cc: add x20, x20, #0x908 0.01 : ffff80001071f2d0: ldr x2, [x29, #104] 0.00 : ffff80001071f2d4: ldr x1, [x20] 0.00 : ffff80001071f2d8: eor x1, x2, x1 0.00 : ffff80001071f2dc: cbnz x1, ffff80001071f360 0.21 : ffff80001071f2e0: ldr x20, [sp, #24] 0.00 : ffff80001071f2e4: ldp x29, x30, [sp], #112 0.00 : ffff80001071f2e8: ret : arm_smmu_atc_inv_to_cmd(): : page_end = (iova + size - 1) >> inval_grain_shift; 0.00 : ffff80001071f2ec: sub x2, x2, #0x1 : page_start = iova >> inval_grain_shift; 0.00 : ffff80001071f2f0: lsr x3, x1, #12 : page_end = (iova + size - 1) >> inval_grain_shift; 0.00 : ffff80001071f2f4: add x1, x2, x1 : fls64(): : return fls(x); : } : #elif BITS_PER_LONG == 64 : static __always_inline int fls64(__u64 x) : { : if (x == 0) 0.00 : ffff80001071f2f8: mov w2, #0x0 // #0 : arm_smmu_atc_inv_to_cmd(): 0.00 : ffff80001071f2fc: lsr x1, x1, #12 : fls64(): 0.00 : ffff80001071f300: cmp x3, x1 0.00 : ffff80001071f304: b.eq ffff80001071f32c // b.none : arm_smmu_atc_inv_to_cmd(): : log2_span = fls_long(page_start ^ page_end); 0.00 : ffff80001071f308: eor x1, x3, x1 : __fls(): : * : * Undefined if no set bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __fls(unsigned long word) : { : return (sizeof(word) * 8) - 1 - __builtin_clzl(word); 0.00 : ffff80001071f30c: mov x2, #0x3f // #63 0.00 : ffff80001071f310: clz x1, x1 0.00 : ffff80001071f314: mov x4, #0xffffffffffffffff // #-1 0.00 : ffff80001071f318: sub x1, x2, x1 : fls64(): : return 0; : return __fls(x) + 1; 0.00 : ffff80001071f31c: add w1, w1, #0x1 0.00 : ffff80001071f320: and w2, w1, #0xff 0.00 : ffff80001071f324: lsl x1, x4, x1 0.00 : ffff80001071f328: and x1, x1, x3 : arm_smmu_atc_inv_to_cmd(): : cmd->atc.addr = page_start << inval_grain_shift; 0.00 : ffff80001071f32c: lsl x1, x1, #12 0.00 : ffff80001071f330: str x1, [x29, #88] : cmd->atc.size = log2_span; 0.00 : ffff80001071f334: strb w2, [x29, #96] 0.00 : ffff80001071f338: b ffff80001071f238 : spin_unlock_irqrestore(): 0.00 : ffff80001071f33c: mov x1, x24 0.00 : ffff80001071f340: mov x0, x21 0.00 : ffff80001071f344: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff80001071f348: b ffff80001071f2b4 : arm_smmu_atc_inv_domain(): : return ret ? -ETIMEDOUT : 0; 0.00 : ffff80001071f34c: mov w0, #0xffffff92 // #-110 0.00 : ffff80001071f350: ldr x19, [x29, #16] 0.00 : ffff80001071f354: ldp x21, x22, [x29, #32] 0.00 : ffff80001071f358: ldp x23, x24, [x29, #48] 0.00 : ffff80001071f35c: b ffff80001071f2cc 0.00 : ffff80001071f360: str x19, [x29, #16] 0.00 : ffff80001071f364: stp x21, x22, [x29, #32] 0.00 : ffff80001071f368: stp x23, x24, [x29, #48] : } 0.00 : ffff80001071f36c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (150077 samples, percent: local period) ------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010716718 <__arm_lpae_unmap>: : __arm_lpae_unmap(): : : static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, : struct iommu_iotlb_gather *gather, : unsigned long iova, size_t size, int lvl, : arm_lpae_iopte *ptep) : { 0.63 : ffff800010716718: stp x29, x30, [sp, #-160]! : arm_lpae_iopte pte; : struct io_pgtable *iop = &data->iop; : : /* Something went horribly wrong and we ran out of page table */ : if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 0.00 : ffff80001071671c: cmp w4, #0x4 : { 0.00 : ffff800010716720: mov x29, sp : if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 0.00 : ffff800010716724: b.eq ffff800010716a94 <__arm_lpae_unmap+0x37c> // b.none 0.07 : ffff800010716728: str x22, [x29, #40] : return 0; : : ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 0.01 : ffff80001071672c: mov w7, #0x4 // #4 0.18 : ffff800010716730: str x25, [x29, #64] 0.01 : ffff800010716734: sub w7, w7, w4 0.05 : ffff800010716738: ldp w6, w22, [x0, #116] 0.02 : ffff80001071673c: mov w9, w22 0.01 : ffff800010716740: cmp w6, w4 0.00 : ffff800010716744: mul w7, w7, w22 0.62 : ffff800010716748: add w7, w7, #0x3 0.00 : ffff80001071674c: lsr x8, x2, x7 0.03 : ffff800010716750: b.ne ffff800010716758 <__arm_lpae_unmap+0x40> // b.any 0.07 : ffff800010716754: ldr w9, [x0, #112] 0.02 : ffff800010716758: mov w6, #0x1 // #1 0.00 : ffff80001071675c: lsl w6, w6, w9 0.00 : ffff800010716760: sub w6, w6, #0x1 0.12 : ffff800010716764: sxtw x6, w6 0.06 : ffff800010716768: and x6, x6, x8 0.00 : ffff80001071676c: add x11, x5, x6, lsl #3 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.07 : ffff800010716770: ldr x25, [x5, x6, lsl #3] : __arm_lpae_unmap(): : pte = READ_ONCE(*ptep); : if (WARN_ON(!pte)) 0.18 : ffff800010716774: cbz x25, ffff800010716aa4 <__arm_lpae_unmap+0x38c> 89.18 : ffff800010716778: stp x19, x20, [x29, #16] 0.01 : ffff80001071677c: cmp w4, #0x3 0.97 : ffff800010716780: str x21, [x29, #32] 0.00 : ffff800010716784: mov x19, x0 0.00 : ffff800010716788: str x24, [x29, #56] : return 0; : : /* If the size matches this level, we're in the right place */ : if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { 0.00 : ffff80001071678c: mov x0, #0x1 // #1 0.00 : ffff800010716790: stp x26, x27, [x29, #72] 0.00 : ffff800010716794: lsl x7, x0, x7 0.04 : ffff800010716798: str x28, [x29, #88] 0.00 : ffff80001071679c: cset w24, eq // eq = none 0.02 : ffff8000107167a0: mov x26, x3 0.01 : ffff8000107167a4: mov x20, x2 0.80 : ffff8000107167a8: mov w27, w4 0.00 : ffff8000107167ac: mov x21, x1 0.00 : ffff8000107167b0: and x28, x25, #0x3 0.00 : ffff8000107167b4: cmp x7, x3 0.00 : ffff8000107167b8: b.eq ffff8000107169c0 <__arm_lpae_unmap+0x2a8> // b.none : iopte_leaf(): : if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE) 0.00 : ffff8000107167bc: ldr w0, [x19] 0.01 : ffff8000107167c0: cmp w24, #0x0 0.00 : ffff8000107167c4: ccmp w0, #0x5, #0x4, ne // ne = any 0.01 : ffff8000107167c8: b.ne ffff800010716950 <__arm_lpae_unmap+0x238> // b.any : return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK; 0.13 : ffff8000107167cc: cmp x28, #0x1 0.02 : ffff8000107167d0: mov x24, #0x8 // #8 0.05 : ffff8000107167d4: cset w0, eq // eq = none 0.01 : ffff8000107167d8: add w27, w27, #0x1 0.48 : ffff8000107167dc: lsl x24, x24, x22 : __arm_lpae_unmap(): : } else { : io_pgtable_tlb_add_page(iop, gather, iova, size); : } : : return size; : } else if (iopte_leaf(pte, lvl, iop->fmt)) { 0.00 : ffff8000107167e0: cbz w0, ffff800010716968 <__arm_lpae_unmap+0x250> : arm_lpae_split_blk_unmap(): : if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 0.00 : ffff8000107167e4: cmp w27, #0x4 0.00 : ffff8000107167e8: b.eq ffff800010716b54 <__arm_lpae_unmap+0x43c> // b.none : tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg); 0.00 : ffff8000107167ec: ldr x3, [x19, #56] 0.00 : ffff8000107167f0: add x2, x19, #0x28 0.00 : ffff8000107167f4: str x11, [x29, #152] 0.00 : ffff8000107167f8: mov w1, #0xa20 // #2592 0.00 : ffff8000107167fc: mov x0, x24 0.00 : ffff800010716800: bl ffff800010715be8 <__arm_lpae_alloc_pages.isra.22> 0.00 : ffff800010716804: mov x28, x0 : if (!tablep) 0.00 : ffff800010716808: ldr x11, [x29, #152] 0.00 : ffff80001071680c: cbz x0, ffff800010716948 <__arm_lpae_unmap+0x230> : size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 0.00 : ffff800010716810: mov w0, #0x4 // #4 0.00 : ffff800010716814: sub w0, w0, w27 0.00 : ffff800010716818: mov x2, #0x1 // #1 : int i, unmap_idx = -1; 0.00 : ffff80001071681c: mov w5, #0xffffffff // #-1 0.00 : ffff800010716820: ldr w3, [x19, #120] : size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 0.00 : ffff800010716824: mul w22, w22, w0 0.00 : ffff800010716828: add w22, w22, #0x3 0.00 : ffff80001071682c: lsl x2, x2, x22 : if (size == split_sz) 0.00 : ffff800010716830: cmp x26, x2 0.00 : ffff800010716834: b.eq ffff800010716b08 <__arm_lpae_unmap+0x3f0> // b.none : iopte_to_paddr(): : u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716838: and x22, x25, #0xfffffffff000 : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff80001071683c: mov x1, #0x8 // #8 0.00 : ffff800010716840: lsl x1, x1, x3 : arm_lpae_split_blk_unmap(): : pte = iopte_prot(blk_pte); 0.00 : ffff800010716844: and x7, x25, #0x7ffffffffffffc : iopte_to_paddr(): : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716848: orr x0, x22, x22, lsl #36 : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff80001071684c: mov x3, #0xffff // #65535 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716850: and x0, x0, #0xfffffffff0000 0.00 : ffff800010716854: cmp x1, x3 0.00 : ffff800010716858: csel x22, x0, x22, hi // hi = pmore : arm_lpae_split_blk_unmap(): : pte = iopte_prot(blk_pte); 0.00 : ffff80001071685c: and x7, x7, #0xffe0000000000fff : for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { 0.00 : ffff800010716860: lsr x8, x24, #3 0.00 : ffff800010716864: add x3, x19, #0x10 0.00 : ffff800010716868: cbz x8, ffff800010716904 <__arm_lpae_unmap+0x1ec> : __arm_lpae_init_pte(): : pte |= ARM_LPAE_PTE_TYPE_PAGE; 0.00 : ffff80001071686c: orr x10, x7, #0x3 0.00 : ffff800010716870: add x3, x19, #0x10 0.00 : ffff800010716874: orr x7, x7, #0x1 0.00 : ffff800010716878: str x23, [x29, #48] : arm_lpae_split_blk_unmap(): : for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { 0.00 : ffff80001071687c: mov x1, #0x0 // #0 0.00 : ffff800010716880: mov w23, #0x0 // #0 0.00 : ffff800010716884: nop : paddr_to_iopte(): : return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716888: orr x6, x22, x22, lsr #36 : arm_lpae_split_blk_unmap(): : if (i == unmap_idx) 0.00 : ffff80001071688c: cmp w23, w5 : paddr_to_iopte(): : return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716890: and x6, x6, #0xfffffffff000 : arm_lpae_split_blk_unmap(): : for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { 0.00 : ffff800010716894: add w23, w23, #0x1 : if (i == unmap_idx) 0.00 : ffff800010716898: b.eq ffff8000107168f0 <__arm_lpae_unmap+0x1d8> // b.none : __arm_lpae_init_pte(): : if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1) 0.00 : ffff80001071689c: ldr w9, [x19] : arm_lpae_split_blk_unmap(): : __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]); 0.00 : ffff8000107168a0: add x0, x28, x1, lsl #3 : __arm_lpae_init_pte(): : if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1) 0.00 : ffff8000107168a4: cmp w9, #0x5 : pte |= ARM_LPAE_PTE_TYPE_PAGE; 0.00 : ffff8000107168a8: ccmp w27, #0x3, #0x0, ne // ne = any 0.00 : ffff8000107168ac: csel x9, x7, x10, ne // ne = any : pte |= paddr_to_iopte(paddr, data); 0.00 : ffff8000107168b0: orr x6, x6, x9 : __arm_lpae_set_pte(): : *ptep = pte; 0.00 : ffff8000107168b4: str x6, [x28, x1, lsl #3] : if (!cfg->coherent_walk) 0.00 : ffff8000107168b8: ldrb w1, [x3, #24] 0.00 : ffff8000107168bc: cbnz w1, ffff8000107168f0 <__arm_lpae_unmap+0x1d8> : __arm_lpae_sync_pte(ptep, cfg); 0.00 : ffff8000107168c0: ldr x1, [x19, #56] 0.00 : ffff8000107168c4: stp x7, x10, [x29, #104] 0.00 : ffff8000107168c8: str x3, [x29, #120] 0.00 : ffff8000107168cc: str w5, [x29, #132] 0.00 : ffff8000107168d0: stp x2, x11, [x29, #136] 0.00 : ffff8000107168d4: str x8, [x29, #152] 0.00 : ffff8000107168d8: bl ffff800010715a88 <__arm_lpae_sync_pte.isra.21> 0.00 : ffff8000107168dc: ldr w5, [x29, #132] 0.00 : ffff8000107168e0: ldp x7, x10, [x29, #104] 0.00 : ffff8000107168e4: ldr x3, [x29, #120] 0.00 : ffff8000107168e8: ldp x2, x11, [x29, #136] 0.00 : ffff8000107168ec: ldr x8, [x29, #152] : arm_lpae_split_blk_unmap(): : for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { 0.00 : ffff8000107168f0: sxtw x1, w23 0.00 : ffff8000107168f4: add x22, x22, x2 0.00 : ffff8000107168f8: cmp x8, x1 0.00 : ffff8000107168fc: b.hi ffff800010716888 <__arm_lpae_unmap+0x170> // b.pmore 0.00 : ffff800010716900: ldr x23, [x29, #48] 0.00 : ffff800010716904: str w5, [x29, #152] : pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg); 0.00 : ffff800010716908: mov x2, x25 0.00 : ffff80001071690c: mov x1, x11 0.00 : ffff800010716910: mov x0, x28 0.00 : ffff800010716914: bl ffff800010715b10 0.00 : ffff800010716918: mov x22, x0 : if (pte != blk_pte) { 0.00 : ffff80001071691c: cmp x25, x0 0.00 : ffff800010716920: ldr w5, [x29, #152] 0.00 : ffff800010716924: b.eq ffff800010716a68 <__arm_lpae_unmap+0x350> // b.none : __arm_lpae_free_pages(tablep, tablesz, cfg); 0.00 : ffff800010716928: ldrb w2, [x19, #40] 0.00 : ffff80001071692c: mov x0, x28 0.00 : ffff800010716930: add x3, x19, #0x38 0.00 : ffff800010716934: mov x1, x24 0.00 : ffff800010716938: bl ffff800010716518 <__arm_lpae_free_pages.isra.20> : if (iopte_type(pte, lvl - 1) != ARM_LPAE_PTE_TYPE_TABLE) 0.00 : ffff80001071693c: and x0, x22, #0x3 0.00 : ffff800010716940: cmp x0, #0x3 0.00 : ffff800010716944: b.eq ffff800010716ab8 <__arm_lpae_unmap+0x3a0> // b.none : return 0; 0.00 : ffff800010716948: mov x0, #0x0 // #0 0.00 : ffff80001071694c: b ffff8000107169a4 <__arm_lpae_unmap+0x28c> : iopte_leaf(): : return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_PAGE; 0.00 : ffff800010716950: cmp x28, #0x3 0.00 : ffff800010716954: mov x24, #0x8 // #8 0.00 : ffff800010716958: cset w0, eq // eq = none 0.00 : ffff80001071695c: add w27, w27, #0x1 0.00 : ffff800010716960: lsl x24, x24, x22 : __arm_lpae_unmap(): : } else if (iopte_leaf(pte, lvl, iop->fmt)) { 0.00 : ffff800010716964: cbnz w0, ffff8000107167e4 <__arm_lpae_unmap+0xcc> : iopte_to_paddr(): : u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716968: and x25, x25, #0xfffffffff000 : __arm_lpae_unmap(): : return arm_lpae_split_blk_unmap(data, gather, iova, size, pte, : lvl + 1, ptep); : } : : /* Keep on walkin' */ : ptep = iopte_deref(pte, data); 0.00 : ffff80001071696c: adrp x1, ffff8000112f2000 : iopte_to_paddr(): : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716970: mov x0, #0xffff // #65535 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716974: cmp x24, x0 0.87 : ffff800010716978: orr x0, x25, x25, lsl #36 : __arm_lpae_unmap(): : ptep = iopte_deref(pte, data); 0.00 : ffff80001071697c: ldr x5, [x1, #1872] : iopte_to_paddr(): : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716980: and x0, x0, #0xfffffffff0000 : __arm_lpae_unmap(): : return __arm_lpae_unmap(data, gather, iova, size, lvl + 1, ptep); 0.00 : ffff800010716984: mov w4, w27 : iopte_to_paddr(): : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716988: csel x25, x0, x25, hi // hi = pmore : __arm_lpae_unmap(): : return __arm_lpae_unmap(data, gather, iova, size, lvl + 1, ptep); 0.13 : ffff80001071698c: mov x3, x26 0.00 : ffff800010716990: sub x5, x25, x5 0.00 : ffff800010716994: mov x2, x20 0.00 : ffff800010716998: mov x1, x21 0.00 : ffff80001071699c: mov x0, x19 0.00 : ffff8000107169a0: bl ffff800010716718 <__arm_lpae_unmap> 0.23 : ffff8000107169a4: ldp x19, x20, [x29, #16] 1.06 : ffff8000107169a8: ldp x21, x22, [x29, #32] 0.15 : ffff8000107169ac: ldp x24, x25, [x29, #56] 1.44 : ffff8000107169b0: ldp x26, x27, [x29, #72] 1.16 : ffff8000107169b4: ldr x28, [x29, #88] : } 0.23 : ffff8000107169b8: ldp x29, x30, [sp], #160 0.00 : ffff8000107169bc: ret : __arm_lpae_set_pte(): : *ptep = pte; 0.00 : ffff8000107169c0: str xzr, [x5, x6, lsl #3] : if (!cfg->coherent_walk) 0.03 : ffff8000107169c4: ldrb w0, [x19, #40] 0.00 : ffff8000107169c8: cbz w0, ffff800010716a58 <__arm_lpae_unmap+0x340> : iopte_leaf(): : if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE) 0.21 : ffff8000107169cc: ldr w0, [x19] 0.00 : ffff8000107169d0: cmp w24, #0x0 0.00 : ffff8000107169d4: ccmp w0, #0x5, #0x4, ne // ne = any 0.00 : ffff8000107169d8: b.eq ffff800010716a4c <__arm_lpae_unmap+0x334> // b.none : return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_PAGE; 0.01 : ffff8000107169dc: cmp x28, #0x3 0.00 : ffff8000107169e0: cset w0, eq // eq = none : __arm_lpae_unmap(): : if (!iopte_leaf(pte, lvl, iop->fmt)) { 0.00 : ffff8000107169e4: cbnz w0, ffff800010716b40 <__arm_lpae_unmap+0x428> : io_pgtable_tlb_flush_walk(): : : static inline void : io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova, : size_t size, size_t granule) : { : iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie); 0.00 : ffff8000107169e8: ldr x4, [x19, #48] : __arm_lpae_unmap(): : io_pgtable_tlb_flush_walk(iop, iova, size, 0.00 : ffff8000107169ec: mov x21, #0x8 // #8 : io_pgtable_tlb_flush_walk(): 0.00 : ffff8000107169f0: ldr x3, [x19, #8] 0.00 : ffff8000107169f4: mov x1, x26 : __arm_lpae_unmap(): 0.00 : ffff8000107169f8: ldr w2, [x19, #120] : io_pgtable_tlb_flush_walk(): 0.00 : ffff8000107169fc: mov x0, x20 0.00 : ffff800010716a00: ldr x4, [x4, #8] : iopte_to_paddr(): : u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716a04: and x25, x25, #0xfffffffff000 : io_pgtable_tlb_flush_walk(): 0.00 : ffff800010716a08: lsl x2, x21, x2 0.00 : ffff800010716a0c: blr x4 : iopte_to_paddr(): : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716a10: ldr w4, [x19, #120] : __arm_lpae_unmap(): : ptep = iopte_deref(pte, data); 0.00 : ffff800010716a14: adrp x2, ffff8000112f2000 : iopte_to_paddr(): : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716a18: orr x0, x25, x25, lsl #36 : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716a1c: mov x3, #0xffff // #65535 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716a20: and x0, x0, #0xfffffffff0000 : __arm_lpae_unmap(): : ptep = iopte_deref(pte, data); 0.00 : ffff800010716a24: ldr x2, [x2, #1872] : iopte_to_paddr(): : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716a28: lsl x21, x21, x4 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716a2c: cmp x21, x3 0.00 : ffff800010716a30: csel x25, x0, x25, hi // hi = pmore : __arm_lpae_unmap(): : __arm_lpae_free_pgtable(data, lvl + 1, ptep); 0.00 : ffff800010716a34: add w1, w27, #0x1 0.00 : ffff800010716a38: mov x0, x19 0.00 : ffff800010716a3c: sub x2, x25, x2 0.00 : ffff800010716a40: bl ffff8000107165d8 <__arm_lpae_free_pgtable> : io_pgtable_tlb_add_page(): : static inline void : io_pgtable_tlb_add_page(struct io_pgtable *iop, : struct iommu_iotlb_gather * gather, unsigned long iova, : size_t granule) : { : if (iop->cfg.tlb->tlb_add_page) 0.00 : ffff800010716a44: mov x0, x26 0.00 : ffff800010716a48: b ffff8000107169a4 <__arm_lpae_unmap+0x28c> : iopte_leaf(): : return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK; 0.00 : ffff800010716a4c: cmp x28, #0x1 0.00 : ffff800010716a50: cset w0, eq // eq = none 0.00 : ffff800010716a54: b ffff8000107169e4 <__arm_lpae_unmap+0x2cc> : __arm_lpae_set_pte(): : __arm_lpae_sync_pte(ptep, cfg); 0.00 : ffff800010716a58: ldr x1, [x19, #56] 0.00 : ffff800010716a5c: mov x0, x11 0.00 : ffff800010716a60: bl ffff800010715a88 <__arm_lpae_sync_pte.isra.21> 0.00 : ffff800010716a64: b ffff8000107169cc <__arm_lpae_unmap+0x2b4> : arm_lpae_split_blk_unmap(): : } else if (unmap_idx >= 0) { 0.00 : ffff800010716a68: tbnz w5, #31, ffff800010716ae8 <__arm_lpae_unmap+0x3d0> : io_pgtable_tlb_add_page(): 0.30 : ffff800010716a6c: ldr x0, [x19, #48] : arm_lpae_split_blk_unmap(): : io_pgtable_tlb_add_page(&data->iop, gather, iova, size); 0.01 : ffff800010716a70: ldr x3, [x19, #8] : io_pgtable_tlb_add_page(): 0.00 : ffff800010716a74: ldr x4, [x0, #24] 0.00 : ffff800010716a78: cbz x4, ffff800010716a44 <__arm_lpae_unmap+0x32c> : iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie); 0.08 : ffff800010716a7c: mov x0, x21 0.00 : ffff800010716a80: mov x2, x26 0.00 : ffff800010716a84: mov x1, x20 0.00 : ffff800010716a88: blr x4 : if (iop->cfg.tlb->tlb_add_page) 0.17 : ffff800010716a8c: mov x0, x26 0.00 : ffff800010716a90: b ffff8000107169a4 <__arm_lpae_unmap+0x28c> : __arm_lpae_unmap(): : if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 0.00 : ffff800010716a94: brk #0x800 : return 0; 0.00 : ffff800010716a98: mov x0, #0x0 // #0 : } 0.00 : ffff800010716a9c: ldp x29, x30, [sp], #160 0.00 : ffff800010716aa0: ret : if (WARN_ON(!pte)) 0.00 : ffff800010716aa4: brk #0x800 : return 0; 0.00 : ffff800010716aa8: mov x0, #0x0 // #0 0.00 : ffff800010716aac: ldr x22, [x29, #40] 0.00 : ffff800010716ab0: ldr x25, [x29, #64] 0.00 : ffff800010716ab4: b ffff8000107169b8 <__arm_lpae_unmap+0x2a0> : iopte_to_paddr(): : u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716ab8: and x22, x22, #0xfffffffff000 : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716abc: ldr w5, [x19, #120] : arm_lpae_split_blk_unmap(): : tablep = iopte_deref(pte, data); 0.00 : ffff800010716ac0: adrp x2, ffff8000112f2000 : iopte_to_paddr(): : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716ac4: mov x1, #0x8 // #8 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716ac8: orr x0, x22, x22, lsl #36 : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716acc: mov x3, #0xffff // #65535 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716ad0: and x0, x0, #0xfffffffff0000 : arm_lpae_split_blk_unmap(): : tablep = iopte_deref(pte, data); 0.00 : ffff800010716ad4: ldr x28, [x2, #1872] : iopte_to_paddr(): : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716ad8: lsl x1, x1, x5 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716adc: cmp x1, x3 0.00 : ffff800010716ae0: csel x22, x0, x22, hi // hi = pmore : arm_lpae_split_blk_unmap(): : tablep = iopte_deref(pte, data); 0.00 : ffff800010716ae4: sub x28, x22, x28 : return __arm_lpae_unmap(data, gather, iova, size, lvl, tablep); 0.00 : ffff800010716ae8: mov x5, x28 0.00 : ffff800010716aec: mov w4, w27 0.00 : ffff800010716af0: mov x3, x26 0.00 : ffff800010716af4: mov x2, x20 0.00 : ffff800010716af8: mov x1, x21 0.00 : ffff800010716afc: mov x0, x19 0.00 : ffff800010716b00: bl ffff800010716718 <__arm_lpae_unmap> 0.00 : ffff800010716b04: b ffff8000107169a4 <__arm_lpae_unmap+0x28c> : unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data); 0.00 : ffff800010716b08: mul w0, w3, w0 0.00 : ffff800010716b0c: ldr w5, [x19, #116] 0.00 : ffff800010716b10: mov w1, w3 0.00 : ffff800010716b14: add w0, w0, #0x3 0.00 : ffff800010716b18: cmp w5, w27 0.00 : ffff800010716b1c: lsr x0, x20, x0 0.00 : ffff800010716b20: b.eq ffff800010716b38 <__arm_lpae_unmap+0x420> // b.none 0.00 : ffff800010716b24: mov w5, #0x1 // #1 0.00 : ffff800010716b28: lsl w5, w5, w1 0.00 : ffff800010716b2c: sub w5, w5, #0x1 0.00 : ffff800010716b30: and w5, w5, w0 0.00 : ffff800010716b34: b ffff800010716838 <__arm_lpae_unmap+0x120> 0.00 : ffff800010716b38: ldr w1, [x19, #112] 0.00 : ffff800010716b3c: b ffff800010716b24 <__arm_lpae_unmap+0x40c> : __arm_lpae_unmap(): : } else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) { 0.01 : ffff800010716b40: ldr x0, [x19, #16] 0.00 : ffff800010716b44: tbz w0, #4, ffff800010716a6c <__arm_lpae_unmap+0x354> : smp_wmb(); 0.00 : ffff800010716b48: dmb ishst : io_pgtable_tlb_add_page(): 0.00 : ffff800010716b4c: mov x0, x26 0.00 : ffff800010716b50: b ffff8000107169a4 <__arm_lpae_unmap+0x28c> : arm_lpae_split_blk_unmap(): : if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 0.00 : ffff800010716b54: brk #0x800 : return 0; 0.00 : ffff800010716b58: mov x0, #0x0 // #0 0.00 : ffff800010716b5c: b ffff8000107169a4 <__arm_lpae_unmap+0x28c> Percent | Source code & Disassembly of vmlinux for cycles (143214 samples, percent: local period) ------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102da3a0 : : aio_complete_rw(): : list_del(&iocb->ki_list); : spin_unlock_irqrestore(&ctx->ctx_lock, flags); : } : : static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) : { 0.53 : ffff8000102da3a0: stp x29, x30, [sp, #-64]! 0.00 : ffff8000102da3a4: mov x29, sp 0.31 : ffff8000102da3a8: stp x19, x20, [sp, #16] 0.00 : ffff8000102da3ac: mov x19, x0 0.01 : ffff8000102da3b0: stp x21, x22, [sp, #32] 0.00 : ffff8000102da3b4: mov x22, x1 : struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw); : : if (!list_empty_careful(&iocb->ki_list)) 0.00 : ffff8000102da3b8: add x0, x0, #0x90 : { 0.00 : ffff8000102da3bc: mov x21, x2 : list_empty_careful(): : * if another CPU could re-list_add() it. : */ : static inline int list_empty_careful(const struct list_head *head) : { : struct list_head *next = head->next; : return (next == head) && (next == head->prev); 0.01 : ffff8000102da3c0: ldr x1, [x19, #144] 0.00 : ffff8000102da3c4: cmp x0, x1 0.00 : ffff8000102da3c8: b.eq ffff8000102da690 // b.none : spinlock_check(): : * Map the spin_lock functions to the raw variants for PREEMPT_RT=n : */ : : static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock) : { : return &lock->rlock; 0.00 : ffff8000102da3cc: ldr x20, [x19, #96] 0.00 : ffff8000102da3d0: add x20, x20, #0x140 : aio_remove_iocb(): : spin_lock_irqsave(&ctx->ctx_lock, flags); 0.00 : ffff8000102da3d4: mov x0, x20 0.00 : ffff8000102da3d8: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : __list_del_entry(): : __list_del(entry->prev, entry->next); 0.00 : ffff8000102da3dc: ldp x4, x3, [x19, #144] : __list_del(): : next->prev = prev; 0.00 : ffff8000102da3e0: str x3, [x4, #8] : list_del(): : entry->next = LIST_POISON1; 0.00 : ffff8000102da3e4: mov x5, #0x100 // #256 : entry->prev = LIST_POISON2; 0.00 : ffff8000102da3e8: mov x2, #0x122 // #290 : entry->next = LIST_POISON1; 0.00 : ffff8000102da3ec: movk x5, #0xdead, lsl #48 : entry->prev = LIST_POISON2; 0.00 : ffff8000102da3f0: movk x2, #0xdead, lsl #48 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000102da3f4: str x4, [x3] : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000102da3f8: mov x1, x0 : list_del(): 0.00 : ffff8000102da3fc: stp x5, x2, [x19, #144] : spin_unlock_irqrestore(): 0.00 : ffff8000102da400: mov x0, x20 0.00 : ffff8000102da404: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : aio_complete_rw(): : aio_remove_iocb(iocb); : : if (kiocb->ki_flags & IOCB_WRITE) { 0.16 : ffff8000102da408: ldr w0, [x19, #32] 0.00 : ffff8000102da40c: tbz w0, #6, ffff8000102da428 : struct inode *inode = file_inode(kiocb->ki_filp); 0.00 : ffff8000102da410: ldr x0, [x19] 0.00 : ffff8000102da414: ldr x1, [x0, #32] : file_end_write(): : return __sb_start_write(file_inode(file)->i_sb, SB_FREEZE_WRITE, false); : } : : static inline void file_end_write(struct file *file) : { : if (!S_ISREG(file_inode(file)->i_mode)) 0.00 : ffff8000102da418: ldrh w0, [x1] 0.00 : ffff8000102da41c: and w0, w0, #0xf000 0.00 : ffff8000102da420: cmp w0, #0x8, lsl #12 0.00 : ffff8000102da424: b.eq ffff8000102da6ec // b.none : iocb_put(): : if (refcount_dec_and_test(&iocb->ki_refcnt)) { 0.41 : ffff8000102da428: add x0, x19, #0xa0 : aio_complete_rw(): : __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE); : file_end_write(kiocb->ki_filp); : } : : iocb->ki_res.res = res; : iocb->ki_res.res2 = res2; 0.09 : ffff8000102da42c: stp x22, x21, [x19, #128] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102da430: b ffff8000102da664 0.12 : ffff8000102da434: b ffff8000102da664 : __lse_atomic_fetch_sub_release(): : return i; \ : } : : ATOMIC_FETCH_OP_SUB(_relaxed, ) : ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") : ATOMIC_FETCH_OP_SUB(_release, l, "memory") 0.10 : ffff8000102da438: mov w1, #0x1 // #1 0.00 : ffff8000102da43c: neg w1, w1 0.00 : ffff8000102da440: ldaddl w1, w1, [x0] : refcount_sub_and_test(): : */ : static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) : { : int old = atomic_fetch_sub_release(i, &r->refs); : : if (old == i) { 29.69 : ffff8000102da444: cmp w1, #0x1 0.00 : ffff8000102da448: b.ne ffff8000102da678 // b.any 0.03 : ffff8000102da44c: stp x23, x24, [x29, #48] : smp_acquire__after_ctrl_dep(); 0.34 : ffff8000102da450: dmb ishld : aio_complete(): : struct kioctx *ctx = iocb->ki_ctx; 3.07 : ffff8000102da454: ldr x20, [x19, #96] : spinlock_check(): : return &lock->rlock; 0.00 : ffff8000102da458: add x22, x20, #0x1c8 : aio_complete(): : spin_lock_irqsave(&ctx->completion_lock, flags); 0.00 : ffff8000102da45c: mov x0, x22 0.01 : ffff8000102da460: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : pos = tail + AIO_EVENTS_OFFSET; 2.74 : ffff8000102da464: ldr w2, [x20, #448] : spin_lock_irqsave(&ctx->completion_lock, flags); 0.73 : ffff8000102da468: mov x23, x0 : if (++tail >= ctx->nr_events) 4.48 : ffff8000102da46c: ldr w4, [x20, #144] : pos = tail + AIO_EVENTS_OFFSET; 0.00 : ffff8000102da470: add w2, w2, #0x1 : ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.03 : ffff8000102da474: ldr x0, [x20, #168] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.01 : ffff8000102da478: mrs x1, sp_el0 : aio_complete(): 0.02 : ffff8000102da47c: lsr w24, w2, #7 : __read_once_size(): : __READ_ONCE_SIZE; 0.17 : ffff8000102da480: ldr w3, [x1, #16] : aio_complete(): : tail = 0; 0.00 : ffff8000102da484: cmp w2, w4 : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000102da488: add w3, w3, #0x1 : aio_complete(): 0.00 : ffff8000102da48c: csel w21, w2, wzr, cc // cc = lo, ul, last : ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.04 : ffff8000102da490: ldr x0, [x0, x24, lsl #3] : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.64 : ffff8000102da494: str w3, [x1, #16] : pagefault_disabled_inc(): : } : #endif : : static __always_inline void pagefault_disabled_inc(void) : { : current->pagefault_disabled++; 0.01 : ffff8000102da498: ldr w3, [x1, #2448] 0.00 : ffff8000102da49c: add w3, w3, #0x1 0.02 : ffff8000102da4a0: str w3, [x1, #2448] : lowmem_page_address(): : */ : #include : : static __always_inline void *lowmem_page_address(const struct page *page) : { : return page_to_virt(page); 0.00 : ffff8000102da4a4: mov x3, #0x200000 // #2097152 : aio_complete(): : event = ev_page + pos % AIO_EVENTS_PER_PAGE; 0.00 : ffff8000102da4a8: ubfiz x4, x2, #5, #7 : lowmem_page_address(): 0.00 : ffff8000102da4ac: movk x3, #0x200, lsl #32 0.00 : ffff8000102da4b0: add x0, x0, x3 : aio_complete(): : *event = iocb->ki_res; 0.00 : ffff8000102da4b4: mov x5, #0xffff000000000000 // #-281474976710656 : lowmem_page_address(): 0.00 : ffff8000102da4b8: lsr x0, x0, #6 : aio_complete(): 0.24 : ffff8000102da4bc: ldp x2, x3, [x19, #112] 0.00 : ffff8000102da4c0: add x0, x4, x0, lsl #12 0.00 : ffff8000102da4c4: add x0, x0, x5 0.02 : ffff8000102da4c8: stp x2, x3, [x0] 2.39 : ffff8000102da4cc: ldp x2, x3, [x19, #128] 0.00 : ffff8000102da4d0: stp x2, x3, [x0, #16] : pagefault_disabled_dec(): : } : : static __always_inline void pagefault_disabled_dec(void) : { : current->pagefault_disabled--; 0.34 : ffff8000102da4d4: ldr w0, [x1, #2448] 0.00 : ffff8000102da4d8: sub w0, w0, #0x1 0.08 : ffff8000102da4dc: str w0, [x1, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.02 : ffff8000102da4e0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102da4e4: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.01 : ffff8000102da4e8: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102da4ec: cbnz x0, ffff8000102da6ac : __kunmap_atomic(): : #define kmap_atomic_prot(page, prot) kmap_atomic(page) : : static inline void __kunmap_atomic(void *addr) : { : pagefault_enable(); : preempt_enable(); 0.00 : ffff8000102da4f0: bl ffff800010cd2e50 : aio_complete(): : flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.22 : ffff8000102da4f4: ldr x0, [x20, #168] 0.00 : ffff8000102da4f8: ldr x0, [x0, x24, lsl #3] 0.00 : ffff8000102da4fc: bl ffff8000100a3110 : smp_wmb(); /* make event visible before updating tail */ 0.22 : ffff8000102da500: dmb ishst : ctx->tail = tail; 16.95 : ffff8000102da504: str w21, [x20, #448] : ring = kmap_atomic(ctx->ring_pages[0]); 0.00 : ffff8000102da508: ldr x1, [x20, #168] : get_current(): 0.00 : ffff8000102da50c: mrs x0, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102da510: ldr w2, [x0, #16] : aio_complete(): 0.23 : ffff8000102da514: ldr x1, [x1] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102da518: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102da51c: str w2, [x0, #16] : pagefault_disabled_inc(): : current->pagefault_disabled++; 0.00 : ffff8000102da520: ldr w2, [x0, #2448] 0.00 : ffff8000102da524: add w2, w2, #0x1 0.00 : ffff8000102da528: str w2, [x0, #2448] : lowmem_page_address(): 0.00 : ffff8000102da52c: mov x2, #0x200000 // #2097152 0.00 : ffff8000102da530: movk x2, #0x200, lsl #32 0.00 : ffff8000102da534: add x1, x1, x2 0.24 : ffff8000102da538: mov x2, #0xffff000000000000 // #-281474976710656 0.00 : ffff8000102da53c: lsr x1, x1, #6 0.00 : ffff8000102da540: add x1, x2, x1, lsl #12 : aio_complete(): : head = ring->head; 0.00 : ffff8000102da544: ldr w24, [x1, #8] : ring->tail = tail; 0.02 : ffff8000102da548: str w21, [x1, #12] : pagefault_disabled_dec(): : current->pagefault_disabled--; 0.12 : ffff8000102da54c: ldr w1, [x0, #2448] 0.00 : ffff8000102da550: sub w1, w1, #0x1 0.36 : ffff8000102da554: str w1, [x0, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.13 : ffff8000102da558: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102da55c: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.17 : ffff8000102da560: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102da564: cbnz x1, ffff8000102da6a0 : __kunmap_atomic(): 0.00 : ffff8000102da568: bl ffff800010cd2e50 : aio_complete(): : flush_dcache_page(ctx->ring_pages[0]); 0.22 : ffff8000102da56c: ldr x0, [x20, #168] 0.00 : ffff8000102da570: ldr x0, [x0] 0.00 : ffff8000102da574: bl ffff8000100a3110 : ctx->completed_events++; 0.22 : ffff8000102da578: ldr w0, [x20, #452] 0.00 : ffff8000102da57c: add w0, w0, #0x1 0.31 : ffff8000102da580: str w0, [x20, #452] : if (ctx->completed_events > 1) 0.00 : ffff8000102da584: cmp w0, #0x1 0.00 : ffff8000102da588: b.hi ffff8000102da6b8 // b.pmore : spin_unlock_irqrestore(): : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000102da58c: mov x0, x22 0.00 : ffff8000102da590: mov x1, x23 0.00 : ffff8000102da594: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : aio_complete(): : if (iocb->ki_eventfd) 0.00 : ffff8000102da598: ldr x0, [x19, #168] 0.00 : ffff8000102da59c: cbz x0, ffff8000102da5a8 : eventfd_signal(iocb->ki_eventfd, 1); 0.00 : ffff8000102da5a0: mov x1, #0x1 // #1 0.00 : ffff8000102da5a4: bl ffff8000102d81c0 : smp_mb(); 0.01 : ffff8000102da5a8: dmb ish : __read_once_size(): : __READ_ONCE_SIZE; 2.88 : ffff8000102da5ac: ldr x1, [x20, #424] : waitqueue_active(): : * Also note that this 'optimization' trades a spin_lock() for an smp_mb(), : * which (when the lock is uncontended) are of roughly equal cost. : */ : static inline int waitqueue_active(struct wait_queue_head *wq_head) : { : return !list_empty(&wq_head->head); 0.00 : ffff8000102da5b0: add x2, x20, #0x1a8 0.00 : ffff8000102da5b4: add x0, x20, #0x1a0 : aio_complete(): : if (waitqueue_active(&ctx->wait)) 0.00 : ffff8000102da5b8: cmp x2, x1 7.54 : ffff8000102da5bc: b.eq ffff8000102da5d0 // b.none : wake_up(&ctx->wait); 0.03 : ffff8000102da5c0: mov x3, #0x0 // #0 0.00 : ffff8000102da5c4: mov w2, #0x1 // #1 0.00 : ffff8000102da5c8: mov w1, #0x3 // #3 0.00 : ffff8000102da5cc: bl ffff80001012f9c0 <__wake_up> : iocb_destroy(): : if (iocb->ki_eventfd) 0.33 : ffff8000102da5d0: ldr x0, [x19, #168] 0.00 : ffff8000102da5d4: cbz x0, ffff8000102da5dc : eventfd_ctx_put(iocb->ki_eventfd); 0.00 : ffff8000102da5d8: bl ffff8000102d7d50 : if (iocb->ki_filp) 1.29 : ffff8000102da5dc: ldr x0, [x19] 0.00 : ffff8000102da5e0: cbz x0, ffff8000102da5e8 : fput(iocb->ki_filp); 0.24 : ffff8000102da5e4: bl ffff80001027df98 : percpu_ref_put(&iocb->ki_ctx->reqs); 0.63 : ffff8000102da5e8: ldr x20, [x19, #96] : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000102da5ec: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.01 : ffff8000102da5f0: ldr x0, [x20, #72] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000102da5f4: tst x0, #0x3 0.00 : ffff8000102da5f8: b.ne ffff8000102da714 // b.any : get_current(): 0.67 : ffff8000102da5fc: mrs x1, sp_el0 : __read_once_size(): 0.06 : ffff8000102da600: ldr w2, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102da604: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.22 : ffff8000102da608: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102da60c: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000102da610: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.03 : ffff8000102da614: add x0, x0, x2 0.04 : ffff8000102da618: ldxr x5, [x0] 6.20 : ffff8000102da61c: add x5, x5, x3 0.01 : ffff8000102da620: stxr w4, x5, [x0] 0.00 : ffff8000102da624: cbnz w4, ffff8000102da618 : __read_once_size(): : __READ_ONCE_SIZE; 0.01 : ffff8000102da628: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102da62c: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.30 : ffff8000102da630: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102da634: cbnz x0, ffff8000102da6e0 : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff8000102da638: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.23 : ffff8000102da63c: bl ffff800010160d20 <__rcu_read_unlock> : iocb_destroy(): : kmem_cache_free(kiocb_cachep, iocb); 0.23 : ffff8000102da640: adrp x0, ffff800011b33000 0.00 : ffff8000102da644: mov x1, x19 0.05 : ffff8000102da648: ldr x0, [x0, #424] 0.00 : ffff8000102da64c: bl ffff800010253518 : aio_complete_rw(): : iocb_put(iocb); : } 0.03 : ffff8000102da650: ldp x23, x24, [x29, #48] 0.24 : ffff8000102da654: ldp x19, x20, [sp, #16] 0.00 : ffff8000102da658: ldp x21, x22, [sp, #32] 0.01 : ffff8000102da65c: ldp x29, x30, [sp], #64 0.00 : ffff8000102da660: ret : __ll_sc_atomic_fetch_sub_release(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000102da664: mov w2, #0x1 // #1 0.00 : ffff8000102da668: add x5, x19, #0xa0 0.00 : ffff8000102da66c: b ffff8000102dd9cc <__arm64_compat_sys_io_pgetevents_time64+0x35c> : refcount_sub_and_test(): : if (old == i) { 0.00 : ffff8000102da670: cmp w1, #0x1 0.00 : ffff8000102da674: b.eq ffff8000102da44c // b.none : return true; : } : : if (unlikely(old < 0 || old - i < 0)) 0.00 : ffff8000102da678: cmp w1, #0x0 0.00 : ffff8000102da67c: b.le ffff8000102da6fc : aio_complete_rw(): 0.00 : ffff8000102da680: ldp x19, x20, [sp, #16] 0.00 : ffff8000102da684: ldp x21, x22, [sp, #32] 0.00 : ffff8000102da688: ldp x29, x30, [sp], #64 0.00 : ffff8000102da68c: ret : list_empty_careful(): : return (next == head) && (next == head->prev); 3.37 : ffff8000102da690: ldr x1, [x19, #152] 0.00 : ffff8000102da694: cmp x0, x1 0.00 : ffff8000102da698: b.ne ffff8000102da3cc // b.any 0.34 : ffff8000102da69c: b ffff8000102da408 : __read_once_size(): : __READ_ONCE_SIZE; 0.16 : ffff8000102da6a0: ldr x0, [x0, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102da6a4: cbz x0, ffff8000102da568 2.61 : ffff8000102da6a8: b ffff8000102da56c : __read_once_size(): 0.20 : ffff8000102da6ac: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102da6b0: cbz x0, ffff8000102da4f0 0.44 : ffff8000102da6b4: b ffff8000102da4f4 : aio_complete(): : refill_reqs_available(ctx, head, tail); 0.01 : ffff8000102da6b8: mov w2, w21 0.00 : ffff8000102da6bc: mov w1, w24 0.00 : ffff8000102da6c0: mov x0, x20 0.00 : ffff8000102da6c4: bl ffff8000102d8f58 : spin_unlock_irqrestore(): 3.86 : ffff8000102da6c8: mov x0, x22 0.00 : ffff8000102da6cc: mov x1, x23 0.00 : ffff8000102da6d0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : aio_complete(): : if (iocb->ki_eventfd) 0.01 : ffff8000102da6d4: ldr x0, [x19, #168] 0.00 : ffff8000102da6d8: cbnz x0, ffff8000102da5a0 0.11 : ffff8000102da6dc: b ffff8000102da5a8 : __read_once_size(): 0.01 : ffff8000102da6e0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102da6e4: cbz x0, ffff8000102da638 0.30 : ffff8000102da6e8: b ffff8000102da63c : file_end_write(): : return; : __sb_end_write(file_inode(file)->i_sb, SB_FREEZE_WRITE); 0.00 : ffff8000102da6ec: ldr x0, [x1, #40] 0.00 : ffff8000102da6f0: mov w1, #0x1 // #1 0.00 : ffff8000102da6f4: bl ffff80001027e6c8 <__sb_end_write> 0.00 : ffff8000102da6f8: b ffff8000102da428 : refcount_sub_and_test(): : refcount_warn_saturate(r, REFCOUNT_SUB_UAF); 0.00 : ffff8000102da6fc: mov w1, #0x3 // #3 0.00 : ffff8000102da700: bl ffff8000104915e0 : aio_complete_rw(): : } 0.00 : ffff8000102da704: ldp x19, x20, [sp, #16] 0.00 : ffff8000102da708: ldp x21, x22, [sp, #32] 0.00 : ffff8000102da70c: ldp x29, x30, [sp], #64 0.00 : ffff8000102da710: ret : iocb_destroy(): : percpu_ref_put(&iocb->ki_ctx->reqs); 0.00 : ffff8000102da714: add x0, x20, #0x40 : arch_static_branch_jump(): 0.00 : ffff8000102da718: b ffff8000102da740 0.00 : ffff8000102da71c: b ffff8000102da740 : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000102da720: mov x1, #0x1 // #1 0.00 : ffff8000102da724: neg x1, x1 0.00 : ffff8000102da728: ldaddal x1, x2, [x0] 0.00 : ffff8000102da72c: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000102da730: cbnz x1, ffff8000102da63c : ref->release(ref); 0.00 : ffff8000102da734: ldr x1, [x0, #16] 0.00 : ffff8000102da738: blr x1 0.00 : ffff8000102da73c: b ffff8000102da63c : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff8000102da740: mov x2, #0x1 // #1 0.00 : ffff8000102da744: add x4, x20, #0x40 0.00 : ffff8000102da748: b ffff8000102dd9e4 <__arm64_compat_sys_io_pgetevents_time64+0x374> 0.00 : ffff8000102da74c: b ffff8000102da730 Percent | Source code & Disassembly of vmlinux for cycles (120861 samples, percent: local period) ------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd8640 <_raw_spin_lock_irqsave>: : _raw_spin_lock_irqsave(): : EXPORT_SYMBOL(_raw_spin_lock); : #endif : : #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE : unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) : { 0.48 : ffff800010cd8640: stp x29, x30, [sp, #-32]! 0.00 : ffff800010cd8644: mov x3, x0 0.00 : ffff800010cd8648: mov x29, sp 0.10 : ffff800010cd864c: str x19, [sp, #16] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.82 : ffff800010cd8650: mrs x19, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.45 : ffff800010cd8654: and w0, w19, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff800010cd8658: cbnz w0, ffff800010cd8664 <_raw_spin_lock_irqsave+0x24> : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.23 : ffff800010cd865c: mov x0, #0x60 // #96 0.08 : ffff800010cd8660: msr daifset, #0x2 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 17.48 : ffff800010cd8664: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.41 : ffff800010cd8668: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.01 : ffff800010cd866c: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 16.66 : ffff800010cd8670: str w0, [x1, #16] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010cd8674: b ffff800010cd86a8 <_raw_spin_lock_irqsave+0x68> 0.13 : ffff800010cd8678: b ffff800010cd86a8 <_raw_spin_lock_irqsave+0x68> : __lse__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, ) : __CMPXCHG_CASE(w, , , 32, ) : __CMPXCHG_CASE(x, , , 64, ) : __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") : __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 0.00 : ffff800010cd867c: mov w1, #0x0 // #0 0.00 : ffff800010cd8680: mov x0, x3 0.00 : ffff800010cd8684: mov w2, #0x1 // #1 0.00 : ffff800010cd8688: mov w4, w1 0.01 : ffff800010cd868c: casa w4, w2, [x3] 60.79 : ffff800010cd8690: mov w0, w4 : atomic_try_cmpxchg_acquire(): : static inline bool : atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) : { : int r, o = *old; : r = atomic_cmpxchg_acquire(v, o, new); : if (unlikely(r != o)) 0.00 : ffff800010cd8694: cbnz w0, ffff800010cd86b8 <_raw_spin_lock_irqsave+0x78> : _raw_spin_lock_irqsave(): : return __raw_spin_lock_irqsave(lock); : } 0.00 : ffff800010cd8698: mov x0, x19 1.21 : ffff800010cd869c: ldr x19, [sp, #16] 0.14 : ffff800010cd86a0: ldp x29, x30, [sp], #32 0.00 : ffff800010cd86a4: ret : __ll_sc__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, , , , , K) : __CMPXCHG_CASE(w, , , 32, , , , , K) : __CMPXCHG_CASE( , , , 64, , , , , L) : __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) : __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 0.00 : ffff800010cd86a8: mov x1, #0x0 // #0 0.00 : ffff800010cd86ac: mov w2, #0x1 // #1 0.00 : ffff800010cd86b0: b ffff800010cd8944 <_raw_read_lock_irqsave+0x26c> : atomic_try_cmpxchg_acquire(): 0.00 : ffff800010cd86b4: cbz w0, ffff800010cd8698 <_raw_spin_lock_irqsave+0x58> : queued_spin_lock(): : u32 val = 0; : : if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL))) : return; : : queued_spin_lock_slowpath(lock, val); 0.00 : ffff800010cd86b8: mov w1, w0 0.00 : ffff800010cd86bc: mov x0, x3 0.00 : ffff800010cd86c0: bl ffff800010139e90 : _raw_spin_lock_irqsave(): 0.00 : ffff800010cd86c4: mov x0, x19 0.00 : ffff800010cd86c8: ldr x19, [sp, #16] 0.00 : ffff800010cd86cc: ldp x29, x30, [sp], #32 0.00 : ffff800010cd86d0: ret Percent | Source code & Disassembly of vmlinux for cycles (111459 samples, percent: local period) ------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001071e4e8 <__arm_smmu_cmdq_poll_set_valid_map.isra.33>: : __arm_smmu_cmdq_poll_set_valid_map(): : struct arm_smmu_ll_queue llq = { : .max_n_shift = cmdq->q.llq.max_n_shift, : .prod = sprod, : }; : : ewidx = BIT_WORD(Q_IDX(&llq, eprod)); 0.35 : ffff80001071e4e8: mov w5, #0x1 // #1 : ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG; : : while (llq.prod != eprod) { 0.00 : ffff80001071e4ec: cmp w3, w2 : ewidx = BIT_WORD(Q_IDX(&llq, eprod)); 0.00 : ffff80001071e4f0: lsl w0, w5, w0 0.00 : ffff80001071e4f4: sub w15, w0, #0x1 0.00 : ffff80001071e4f8: and w11, w15, w3 : while (llq.prod != eprod) { 0.00 : ffff80001071e4fc: b.eq ffff80001071e608 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x120> // b.none 0.10 : ffff80001071e500: lsr w16, w11, #6 0.00 : ffff80001071e504: and w11, w11, #0x3f : static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq, 0.53 : ffff80001071e508: stp x29, x30, [sp, #-16]! 0.00 : ffff80001071e50c: mov w14, #0x40 // #64 0.00 : ffff80001071e510: sub w5, w14, w11 0.00 : ffff80001071e514: mov x13, #0xffffffffffffffff // #-1 0.18 : ffff80001071e518: mov x29, sp : swidx = BIT_WORD(Q_IDX(&llq, llq.prod)); : sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG; : : ptr = &cmdq->valid_map[swidx]; : : if ((swidx == ewidx) && (sbidx < ebidx)) 0.00 : ffff80001071e51c: mov x18, x13 : static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq, 0.00 : ffff80001071e520: and w4, w4, #0xff 0.02 : ffff80001071e524: orr w12, w15, w0 0.00 : ffff80001071e528: lsr x13, x13, x5 : limit = ebidx; : : mask = GENMASK(limit - 1, sbidx); 0.00 : ffff80001071e52c: mov x17, #0x1 // #1 0.07 : ffff80001071e530: b ffff80001071e560 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x78> : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.02 : ffff80001071e534: b ffff80001071e600 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x118> 0.05 : ffff80001071e538: b ffff80001071e600 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x118> : __lse_atomic64_xor(): : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) 0.24 : ffff80001071e53c: steor x7, [x5] : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 4.98 : ffff80001071e540: and w5, w2, w12 : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.00 : ffff80001071e544: and w6, w2, #0x80000000 : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.00 : ffff80001071e548: sub w2, w5, w9 0.00 : ffff80001071e54c: add w2, w2, w30 : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.29 : ffff80001071e550: and w2, w2, w12 0.00 : ffff80001071e554: orr w2, w2, w6 : __arm_smmu_cmdq_poll_set_valid_map(): : while (llq.prod != eprod) { 0.00 : ffff80001071e558: cmp w3, w2 0.00 : ffff80001071e55c: b.eq ffff80001071e5f8 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x110> // b.none : swidx = BIT_WORD(Q_IDX(&llq, llq.prod)); 0.07 : ffff80001071e560: and w5, w15, w2 : ptr = &cmdq->valid_map[swidx]; 0.25 : ffff80001071e564: ldr x8, [x1] : sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG; 0.07 : ffff80001071e568: and w9, w5, #0x3f : swidx = BIT_WORD(Q_IDX(&llq, llq.prod)); 18.60 : ffff80001071e56c: lsr w6, w5, #6 : if ((swidx == ewidx) && (sbidx < ebidx)) 0.07 : ffff80001071e570: cmp w16, w6 : mask = GENMASK(limit - 1, sbidx); 29.29 : ffff80001071e574: lsl x7, x17, x9 : if ((swidx == ewidx) && (sbidx < ebidx)) 0.00 : ffff80001071e578: ccmp w11, w9, #0x0, eq // eq = none : mask = GENMASK(limit - 1, sbidx); 0.05 : ffff80001071e57c: neg x7, x7 : if ((swidx == ewidx) && (sbidx < ebidx)) 0.00 : ffff80001071e580: csel x10, x13, x18, hi // hi = pmore 0.00 : ffff80001071e584: csel w30, w11, w14, hi // hi = pmore : mask = GENMASK(limit - 1, sbidx); 0.00 : ffff80001071e588: and x7, x7, x10 : ptr = &cmdq->valid_map[swidx]; 0.00 : ffff80001071e58c: add x5, x8, x6, lsl #3 : * The valid bit is the inverse of the wrap bit. This means : * that a zero-initialised queue is invalid and, after marking : * all entries as valid, they become invalid again when we : * wrap. : */ : if (set) { 0.00 : ffff80001071e590: cbnz w4, ffff80001071e534 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x4c> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.04 : ffff80001071e594: ldr x6, [x8, x6, lsl #3] : __arm_smmu_cmdq_poll_set_valid_map(): : atomic_long_xor(mask, ptr); : } else { /* Poll */ : unsigned long valid; : : valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask; 0.00 : ffff80001071e598: tst w2, w0 0.00 : ffff80001071e59c: csel x10, x7, xzr, eq // eq = none : atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid); 0.06 : ffff80001071e5a0: and x8, x7, x6 1.10 : ffff80001071e5a4: cmp x10, x8 0.00 : ffff80001071e5a8: b.eq ffff80001071e540 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x58> // b.none 0.40 : ffff80001071e5ac: nop : __cmpwait_case_64(): : } : : __CMPWAIT_CASE(w, b, 8); : __CMPWAIT_CASE(w, h, 16); : __CMPWAIT_CASE(w, , 32); : __CMPWAIT_CASE( , , 64); 2.73 : ffff80001071e5b0: sevl 0.03 : ffff80001071e5b4: wfe 5.82 : ffff80001071e5b8: ldxr x8, [x5] 0.00 : ffff80001071e5bc: eor x8, x8, x6 0.00 : ffff80001071e5c0: cbnz x8, ffff80001071e5c8 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0xe0> 1.55 : ffff80001071e5c4: wfe : __read_once_size(): 21.34 : ffff80001071e5c8: ldr x6, [x5] : __arm_smmu_cmdq_poll_set_valid_map(): 0.00 : ffff80001071e5cc: and x8, x7, x6 0.00 : ffff80001071e5d0: cmp x10, x8 0.00 : ffff80001071e5d4: b.ne ffff80001071e5b0 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0xc8> // b.any : queue_inc_prod_n(): : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 11.27 : ffff80001071e5d8: and w5, w2, w12 : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.00 : ffff80001071e5dc: and w6, w2, #0x80000000 : u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n; 0.06 : ffff80001071e5e0: sub w2, w5, w9 0.00 : ffff80001071e5e4: add w2, w2, w30 : return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); 0.13 : ffff80001071e5e8: and w2, w2, w12 0.00 : ffff80001071e5ec: orr w2, w2, w6 : __arm_smmu_cmdq_poll_set_valid_map(): : while (llq.prod != eprod) { 0.06 : ffff80001071e5f0: cmp w3, w2 0.00 : ffff80001071e5f4: b.ne ffff80001071e560 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x78> // b.any : } : : llq.prod = queue_inc_prod_n(&llq, limit - sbidx); : } : } 0.19 : ffff80001071e5f8: ldp x29, x30, [sp], #16 0.00 : ffff80001071e5fc: ret : __ll_sc_atomic64_xor(): : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) : ATOMIC64_OPS(xor, eor, L) 0.00 : ffff80001071e600: b ffff800010721768 0.00 : ffff80001071e604: b ffff80001071e540 <__arm_smmu_cmdq_poll_set_valid_map.isra.33+0x58> 0.00 : ffff80001071e608: ret Percent | Source code & Disassembly of vmlinux for cycles (100625 samples, percent: local period) ------------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104b6820 : : gic_handle_irq(): : if (irqs_enabled) : nmi_exit(); : } : : static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) : { 0.00 : ffff8000104b6820: stp x29, x30, [sp, #-64]! 0.00 : ffff8000104b6824: mov x29, sp 0.00 : ffff8000104b6828: stp x19, x20, [sp, #16] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000104b682c: b ffff8000104b685c : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff8000104b6830: nop : gic_read_iar_common(): 0.00 : ffff8000104b6834: mrs x19, s3_0_c12_c12_0 : static inline u64 gic_read_iar_common(void) : { : u64 irqstat; : : irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1); : dsb(sy); 0.00 : ffff8000104b6838: dsb sy : gic_handle_irq(): : u32 irqnr; : : irqnr = gic_read_iar(); 0.00 : ffff8000104b683c: mov w20, w19 : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff8000104b6840: b ffff8000104b6850 : gic_read_rpr(): 0.00 : ffff8000104b6844: mrs x4, s3_0_c12_c11_3 : gic_handle_irq(): : : if (gic_supports_nmi() && 0.00 : ffff8000104b6848: cmp w4, #0x20 0.00 : ffff8000104b684c: b.eq ffff8000104b692c // b.none : arch_static_branch_jump(): 0.00 : ffff8000104b6850: b ffff8000104b68a8 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff8000104b6854: nop 0.00 : ffff8000104b6858: b ffff8000104b68c0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000104b685c: adrp x1, ffff800011af2000 0.00 : ffff8000104b6860: ldr x1, [x1, #1320] : gic_read_iar(): : if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 0.00 : ffff8000104b6864: tst w1, #0x40 0.00 : ffff8000104b6868: b.eq ffff8000104b6834 // b.none : gic_read_iar_cavium_thunderx(): : */ : static inline u64 gic_read_iar_cavium_thunderx(void) : { : u64 irqstat; : : nops(8); 0.00 : ffff8000104b686c: nop 0.00 : ffff8000104b6870: nop 0.00 : ffff8000104b6874: nop 0.00 : ffff8000104b6878: nop 0.00 : ffff8000104b687c: nop 0.00 : ffff8000104b6880: nop 0.00 : ffff8000104b6884: nop 0.00 : ffff8000104b6888: nop 0.00 : ffff8000104b688c: mrs x19, s3_0_c12_c12_0 : irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1); : nops(4); 0.00 : ffff8000104b6890: nop 0.00 : ffff8000104b6894: nop 0.00 : ffff8000104b6898: nop 0.00 : ffff8000104b689c: nop : mb(); 0.00 : ffff8000104b68a0: dsb sy 0.00 : ffff8000104b68a4: b ffff8000104b683c : test_bit(): 0.00 : ffff8000104b68a8: adrp x1, ffff800011af2000 0.00 : ffff8000104b68ac: ldr x1, [x1, #1320] : gic_handle_irq(): : unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) { : gic_handle_nmi(irqnr, regs); : return; : } : : if (gic_prio_masking_enabled()) { 0.00 : ffff8000104b68b0: tbz x1, #42, ffff8000104b68c0 : gic_write_pmr(): : return read_sysreg_s(SYS_ICC_PMR_EL1); : } : : static inline void gic_write_pmr(u32 val) : { : write_sysreg_s(val, SYS_ICC_PMR_EL1); 0.00 : ffff8000104b68b4: mov x1, #0x60 // #96 0.00 : ffff8000104b68b8: msr s3_0_c4_c6_0, x1 : gic_arch_enable_irqs(): : gic_write_pmr(GIC_PRIO_IRQOFF); : } : : static inline void gic_arch_enable_irqs(void) : { : asm volatile ("msr daifclr, #2" : : : "memory"); 0.00 : ffff8000104b68bc: msr daifclr, #0x2 : gic_handle_irq(): : gic_pmr_mask_irqs(); : gic_arch_enable_irqs(); : } : : /* Check for special IDs first */ : if ((irqnr >= 1020 && irqnr <= 1023)) 83.36 : ffff8000104b68c0: sub w4, w20, #0x3fc 0.00 : ffff8000104b68c4: cmp w4, #0x3 0.00 : ffff8000104b68c8: b.ls ffff8000104b6918 // b.plast : return; : : /* Treat anything but SGIs in a uniform way */ : if (likely(irqnr > 15)) { 0.67 : ffff8000104b68cc: cmp w20, #0xf 0.00 : ffff8000104b68d0: b.ls ffff8000104b6a1c // b.plast : arch_static_branch(): 3.64 : ffff8000104b68d4: nop : gic_write_eoir(): : write_sysreg_s(irq, SYS_ICC_EOIR1_EL1); 0.23 : ffff8000104b68d8: and x19, x19, #0xffffffff 0.00 : ffff8000104b68dc: msr s3_0_c12_c12_1, x19 : isb(); 0.00 : ffff8000104b68e0: isb : gic_handle_irq(): : if (static_branch_likely(&supports_deactivate_key)) : gic_write_eoir(irqnr); : else : isb(); : : err = handle_domain_irq(gic_data.domain, irqnr, regs); 10.80 : ffff8000104b68e4: adrp x4, ffff80001190d000 : handle_domain_irq(): : bool lookup, struct pt_regs *regs); : : static inline int handle_domain_irq(struct irq_domain *domain, : unsigned int hwirq, struct pt_regs *regs) : { : return __handle_domain_irq(domain, hwirq, true, regs); 0.00 : ffff8000104b68e8: mov x3, x0 0.00 : ffff8000104b68ec: mov w2, #0x1 // #1 0.00 : ffff8000104b68f0: mov w1, w20 0.18 : ffff8000104b68f4: ldr x0, [x4, #408] 0.00 : ffff8000104b68f8: bl ffff80001014bb20 <__handle_domain_irq> : gic_handle_irq(): : if (err) { 0.06 : ffff8000104b68fc: cbz w0, ffff8000104b6918 : WARN_ONCE(true, "Unexpected interrupt received!\n"); 0.00 : ffff8000104b6900: adrp x1, ffff800011ad8000 0.00 : ffff8000104b6904: add x1, x1, #0x522 0.00 : ffff8000104b6908: ldrb w0, [x1, #1] 0.00 : ffff8000104b690c: cbz w0, ffff8000104b6a44 : gic_deactivate_unhandled(irqnr); 0.00 : ffff8000104b6910: mov w0, w20 0.00 : ffff8000104b6914: bl ffff8000104b5560 : handle_IPI(irqnr, regs); : #else : WARN_ONCE(true, "Unexpected SGI received!\n"); : #endif : } : } 0.33 : ffff8000104b6918: ldp x19, x20, [sp, #16] 0.72 : ffff8000104b691c: ldp x29, x30, [sp], #64 0.01 : ffff8000104b6920: ret : isb(); 0.00 : ffff8000104b6924: isb 0.00 : ffff8000104b6928: b ffff8000104b68e4 0.00 : ffff8000104b692c: str x21, [x29, #32] : gic_handle_nmi(): : bool irqs_enabled = interrupts_enabled(regs); 0.00 : ffff8000104b6930: mov w21, #0x0 // #0 0.00 : ffff8000104b6934: ldr x1, [x0, #264] 0.00 : ffff8000104b6938: tbz w1, #7, ffff8000104b69bc : arch_static_branch(): 0.00 : ffff8000104b693c: nop : gic_write_eoir(): : write_sysreg_s(irq, SYS_ICC_EOIR1_EL1); 0.00 : ffff8000104b6940: and x19, x19, #0xffffffff 0.00 : ffff8000104b6944: msr s3_0_c12_c12_1, x19 : isb(); 0.00 : ffff8000104b6948: isb : gic_handle_nmi(): : err = handle_domain_nmi(gic_data.domain, irqnr, regs); 0.00 : ffff8000104b694c: adrp x3, ffff80001190d000 0.00 : ffff8000104b6950: mov x2, x0 0.00 : ffff8000104b6954: mov w1, w20 0.00 : ffff8000104b6958: ldr x0, [x3, #408] 0.00 : ffff8000104b695c: bl ffff80001014bbd0 : if (err) 0.00 : ffff8000104b6960: cbnz w0, ffff8000104b6a0c : if (irqs_enabled) 0.00 : ffff8000104b6964: cbnz w21, ffff8000104b6970 0.00 : ffff8000104b6968: ldr x21, [x29, #32] 0.00 : ffff8000104b696c: b ffff8000104b6918 : nmi_exit(); 0.00 : ffff8000104b6970: bl ffff800010161040 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000104b6974: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000104b6978: ldr w1, [x0, #16] : gic_handle_nmi(): 0.00 : ffff8000104b697c: tbz w1, #20, ffff8000104b6ab0 : __read_once_size(): 0.00 : ffff8000104b6980: ldr w1, [x0, #16] : __preempt_count_sub(): : } : : static inline void __preempt_count_sub(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc -= val; 0.00 : ffff8000104b6984: sub w1, w1, #0x110, lsl #12 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104b6988: str w1, [x0, #16] : gic_handle_nmi(): 0.00 : ffff8000104b698c: bl ffff80001014aee0 : is_kernel_in_hyp_mode(): : return __boot_cpu_mode[0] != __boot_cpu_mode[1]; : } : : static inline bool is_kernel_in_hyp_mode(void) : { : return read_sysreg(CurrentEL) == CurrentEL_EL2; 0.00 : ffff8000104b6990: mrs x0, currentel : gic_handle_nmi(): 0.00 : ffff8000104b6994: cmp x0, #0x8 0.00 : ffff8000104b6998: b.ne ffff8000104b6968 // b.any 0.00 : ffff8000104b699c: adrp x0, ffff80001151d000 0.00 : ffff8000104b69a0: add x0, x0, #0x58 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000104b69a4: mrs x1, tpidr_el1 : gic_handle_nmi(): 0.00 : ffff8000104b69a8: ldr x0, [x0, x1] 0.00 : ffff8000104b69ac: tbnz w0, #27, ffff8000104b6968 0.00 : ffff8000104b69b0: msr hcr_el2, x0 0.00 : ffff8000104b69b4: ldr x21, [x29, #32] 0.00 : ffff8000104b69b8: b ffff8000104b6918 : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff8000104b69bc: b ffff8000104b6a88 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff8000104b69c0: nop : is_kernel_in_hyp_mode(): 0.00 : ffff8000104b69c4: mrs x1, currentel : gic_handle_nmi(): : nmi_enter(); 0.00 : ffff8000104b69c8: cmp x1, #0x8 0.00 : ffff8000104b69cc: b.eq ffff8000104b6a60 // b.none 0.00 : ffff8000104b69d0: str x0, [x29, #56] 0.00 : ffff8000104b69d4: bl ffff80001014ae80 : get_current(): 0.00 : ffff8000104b69d8: mrs x1, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000104b69dc: ldr w2, [x1, #16] : gic_handle_nmi(): 0.00 : ffff8000104b69e0: ldr x0, [x29, #56] 0.00 : ffff8000104b69e4: tbnz w2, #20, ffff8000104b6aac : __read_once_size(): 0.00 : ffff8000104b69e8: ldr w2, [x1, #16] : gic_handle_nmi(): 0.00 : ffff8000104b69ec: mov w21, #0x1 // #1 0.00 : ffff8000104b69f0: str x0, [x29, #56] : __preempt_count_add(): : pc += val; 0.00 : ffff8000104b69f4: add w2, w2, #0x110, lsl #12 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104b69f8: str w2, [x1, #16] : gic_handle_nmi(): 0.00 : ffff8000104b69fc: bl ffff800010161120 0.00 : ffff8000104b6a00: ldr x0, [x29, #56] : arch_static_branch(): 0.00 : ffff8000104b6a04: nop 0.00 : ffff8000104b6a08: b ffff8000104b6940 : gic_handle_nmi(): : gic_deactivate_unhandled(irqnr); 0.00 : ffff8000104b6a0c: mov w0, w20 0.00 : ffff8000104b6a10: bl ffff8000104b5560 : if (irqs_enabled) 0.00 : ffff8000104b6a14: cbz w21, ffff8000104b6968 0.00 : ffff8000104b6a18: b ffff8000104b6970 0.00 : ffff8000104b6a1c: and x1, x19, #0xffffffff : gic_write_eoir(): 0.00 : ffff8000104b6a20: msr s3_0_c12_c12_1, x1 0.00 : ffff8000104b6a24: isb : arch_static_branch(): 0.00 : ffff8000104b6a28: nop : gic_write_dir(): 0.00 : ffff8000104b6a2c: msr s3_0_c12_c11_1, x1 : isb(); 0.00 : ffff8000104b6a30: isb : gic_handle_irq(): : handle_IPI(irqnr, regs); 0.00 : ffff8000104b6a34: mov x1, x0 0.00 : ffff8000104b6a38: mov w0, w19 0.00 : ffff8000104b6a3c: bl ffff800010096938 0.00 : ffff8000104b6a40: b ffff8000104b6918 : WARN_ONCE(true, "Unexpected interrupt received!\n"); 0.00 : ffff8000104b6a44: mov w2, #0x1 // #1 0.00 : ffff8000104b6a48: adrp x0, ffff8000111ee000 0.00 : ffff8000104b6a4c: strb w2, [x1, #1] 0.00 : ffff8000104b6a50: add x0, x0, #0x1f0 0.00 : ffff8000104b6a54: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff8000104b6a58: brk #0x800 0.00 : ffff8000104b6a5c: b ffff8000104b6910 : gic_handle_nmi(): : nmi_enter(); 0.00 : ffff8000104b6a60: adrp x1, ffff80001151d000 0.00 : ffff8000104b6a64: add x1, x1, #0x58 : __my_cpu_offset(): 0.00 : ffff8000104b6a68: mrs x3, tpidr_el1 : gic_handle_nmi(): 0.00 : ffff8000104b6a6c: mrs x2, hcr_el2 0.00 : ffff8000104b6a70: str x2, [x1, x3] 0.00 : ffff8000104b6a74: tbnz w2, #27, ffff8000104b69d0 0.00 : ffff8000104b6a78: orr x2, x2, #0x8000000 0.00 : ffff8000104b6a7c: msr hcr_el2, x2 0.00 : ffff8000104b6a80: isb 0.00 : ffff8000104b6a84: b ffff8000104b69d0 : test_bit(): 0.00 : ffff8000104b6a88: adrp x1, ffff800011af2000 0.00 : ffff8000104b6a8c: ldr x1, [x1, #1320] : gic_handle_nmi(): : bool irqs_enabled = interrupts_enabled(regs); 0.00 : ffff8000104b6a90: tbz x1, #42, ffff8000104b69c4 0.00 : ffff8000104b6a94: ldr x1, [x0, #296] 0.00 : ffff8000104b6a98: mov w21, #0x0 // #0 0.00 : ffff8000104b6a9c: cmp x1, #0xe0 0.00 : ffff8000104b6aa0: b.eq ffff8000104b69c4 // b.none : arch_static_branch(): 0.00 : ffff8000104b6aa4: nop 0.00 : ffff8000104b6aa8: b ffff8000104b6940 : gic_handle_nmi(): : nmi_enter(); 0.00 : ffff8000104b6aac: brk #0x800 : nmi_exit(); 0.00 : ffff8000104b6ab0: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (94371 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001027ded0 : : fput_many(): : EXPORT_SYMBOL_GPL(flush_delayed_fput); : : static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput); : : void fput_many(struct file *file, unsigned int refs) : { 0.01 : ffff80001027ded0: stp x29, x30, [sp, #-32]! : if (atomic_long_sub_and_test(refs, &file->f_count)) { 0.01 : ffff80001027ded4: mov w1, w1 : { 0.00 : ffff80001027ded8: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001027dedc: b ffff80001027df5c 0.18 : ffff80001027dee0: b ffff80001027df5c 0.01 : ffff80001027dee4: add x2, x0, #0x38 : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001027dee8: neg x1, x1 0.01 : ffff80001027deec: ldaddal x1, x3, [x2] 98.30 : ffff80001027def0: add x1, x1, x3 : fput_many(): : if (atomic_long_sub_and_test(refs, &file->f_count)) { 0.00 : ffff80001027def4: cbnz x1, ffff80001027df54 0.00 : ffff80001027def8: str x19, [x29, #16] 0.00 : ffff80001027defc: mov x19, x0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001027df00: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001027df04: ldr w1, [x0, #16] : fput_many(): : struct task_struct *task = current; : : if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) { 0.00 : ffff80001027df08: tst w1, #0x1fff00 0.00 : ffff80001027df0c: b.ne ffff80001027df34 // b.any 0.00 : ffff80001027df10: ldr w1, [x0, #44] 0.00 : ffff80001027df14: tbnz w1, #21, ffff80001027df34 : init_task_work(): : typedef void (*task_work_func_t)(struct callback_head *); : : static inline void : init_task_work(struct callback_head *twork, task_work_func_t func) : { : twork->func = func; 0.00 : ffff80001027df18: adrp x1, ffff80001027d000 <__arm64_compat_sys_sendfile+0x108> 0.00 : ffff80001027df1c: add x1, x1, #0xae8 0.00 : ffff80001027df20: str x1, [x19, #8] : fput_many(): : init_task_work(&file->f_u.fu_rcuhead, ____fput); : if (!task_work_add(task, &file->f_u.fu_rcuhead, true)) 0.00 : ffff80001027df24: mov w2, #0x1 // #1 0.00 : ffff80001027df28: mov x1, x19 0.00 : ffff80001027df2c: bl ffff80001010a180 0.00 : ffff80001027df30: cbz w0, ffff80001027df50 : llist_add(): : * : * Returns true if the list was empty prior to adding this entry. : */ : static inline bool llist_add(struct llist_node *new, struct llist_head *head) : { : return llist_add_batch(new, new, head); 0.00 : ffff80001027df34: adrp x2, ffff800011b2f000 0.00 : ffff80001027df38: mov x1, x19 0.00 : ffff80001027df3c: add x2, x2, #0xd20 0.00 : ffff80001027df40: mov x0, x19 0.00 : ffff80001027df44: bl ffff80001048db58 : fput_many(): : * task_work_add() will fail. Fall through to delayed : * fput to avoid leaking *file. : */ : } : : if (llist_add(&file->f_u.fu_llist, &delayed_fput_list)) 0.00 : ffff80001027df48: tst w0, #0xff 0.00 : ffff80001027df4c: b.ne ffff80001027df6c // b.any 0.00 : ffff80001027df50: ldr x19, [x29, #16] : schedule_delayed_work(&delayed_fput_work, 1); : } : } 1.47 : ffff80001027df54: ldp x29, x30, [sp], #32 0.00 : ffff80001027df58: ret : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff80001027df5c: add x4, x0, #0x38 0.00 : ffff80001027df60: b ffff80001027e080 <__fput_sync+0xd0> 0.00 : ffff80001027df64: mov x1, x2 0.00 : ffff80001027df68: b ffff80001027def4 : schedule_delayed_work(): : * workqueue. : */ : static inline bool schedule_delayed_work(struct delayed_work *dwork, : unsigned long delay) : { : return queue_delayed_work(system_wq, dwork, delay); 0.00 : ffff80001027df6c: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> : queue_delayed_work(): : return queue_delayed_work_on(WORK_CPU_UNBOUND, wq, dwork, delay); 0.00 : ffff80001027df70: adrp x2, ffff800011930000 0.00 : ffff80001027df74: add x2, x2, #0x600 0.00 : ffff80001027df78: mov x3, #0x1 // #1 0.00 : ffff80001027df7c: ldr x1, [x0, #496] 0.00 : ffff80001027df80: add x2, x2, #0x18 0.00 : ffff80001027df84: mov w0, #0x100 // #256 0.00 : ffff80001027df88: bl ffff800010105988 0.00 : ffff80001027df8c: ldr x19, [x29, #16] : fput_many(): 0.00 : ffff80001027df90: ldp x29, x30, [sp], #32 0.00 : ffff80001027df94: ret Percent | Source code & Disassembly of vmlinux for cycles (77062 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff8000107d4528 : : nvme_irq(): : nvme_ring_cq_doorbell(nvmeq); : return found; : } : : static irqreturn_t nvme_irq(int irq, void *data) : { 0.02 : ffff8000107d4528: stp x29, x30, [sp, #-80]! 0.00 : ffff8000107d452c: mov x29, sp 0.16 : ffff8000107d4530: stp x19, x20, [sp, #16] : : /* : * The rmb/wmb pair ensures we see all updates from a previous run of : * the irq handler, even if that was on another CPU. : */ : rmb(); 0.05 : ffff8000107d4534: dsb ld : nvme_process_cq(): : *start = nvmeq->cq_head; 1.81 : ffff8000107d4538: ldrh w19, [x1, #112] 0.88 : ffff8000107d453c: ldrb w3, [x1, #116] : while (nvme_cqe_pending(nvmeq)) { 0.25 : ffff8000107d4540: ldr x5, [x1, #72] : *start = nvmeq->cq_head; 0.00 : ffff8000107d4544: mov w20, w19 0.00 : ffff8000107d4548: and w4, w3, #0xffff : nvme_cqe_pending(): : return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == 0.00 : ffff8000107d454c: ubfiz x0, x20, #4, #16 : nvme_update_cq_head(): : nvmeq->cq_head++; 0.03 : ffff8000107d4550: add w2, w20, #0x1 : nvme_cqe_pending(): : return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == 0.00 : ffff8000107d4554: add x0, x5, x0 0.00 : ffff8000107d4558: ldrh w0, [x0, #14] : nvme_process_cq(): : while (nvme_cqe_pending(nvmeq)) { 0.00 : ffff8000107d455c: and w0, w0, #0x1 0.00 : ffff8000107d4560: cmp w0, w4 0.00 : ffff8000107d4564: b.ne ffff8000107d459c // b.any : nvme_update_cq_head(): : if (nvmeq->cq_head == nvmeq->q_depth - 1) { 11.60 : ffff8000107d4568: ldrh w0, [x1, #104] 0.01 : ffff8000107d456c: sub w0, w0, #0x1 0.01 : ffff8000107d4570: cmp w20, w0 : nvmeq->cq_head++; 0.02 : ffff8000107d4574: and w20, w2, #0xffff : if (nvmeq->cq_head == nvmeq->q_depth - 1) { 0.79 : ffff8000107d4578: b.eq ffff8000107d46bc // b.none : nvme_cqe_pending(): : return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == 0.03 : ffff8000107d457c: ubfiz x0, x20, #4, #16 : nvme_update_cq_head(): : nvmeq->cq_head++; 0.13 : ffff8000107d4580: strh w20, [x1, #112] : nvme_cqe_pending(): : return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == 0.00 : ffff8000107d4584: add x0, x5, x0 : nvme_update_cq_head(): : nvmeq->cq_head++; 0.01 : ffff8000107d4588: add w2, w20, #0x1 : nvme_cqe_pending(): : return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == 0.10 : ffff8000107d458c: ldrh w0, [x0, #14] : nvme_process_cq(): : while (nvme_cqe_pending(nvmeq)) { 1.48 : ffff8000107d4590: and w0, w0, #0x1 1.18 : ffff8000107d4594: cmp w0, w4 0.59 : ffff8000107d4598: b.eq ffff8000107d4568 // b.none : if (*start != *end) 0.33 : ffff8000107d459c: cmp w20, w19 0.02 : ffff8000107d45a0: b.eq ffff8000107d45f4 // b.none : nvme_ring_cq_doorbell(): : if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db, 0.07 : ffff8000107d45a4: ldr x2, [x1, #136] : nvme_dbbuf_update_and_check_event(): : if (dbbuf_db) { 0.00 : ffff8000107d45a8: cbz x2, ffff8000107d45dc : nvme_ring_cq_doorbell(): : nvmeq->dbbuf_cq_ei)) 0.00 : ffff8000107d45ac: ldr x3, [x1, #152] : nvme_dbbuf_update_and_check_event(): : wmb(); 0.00 : ffff8000107d45b0: dsb st : old_value = *dbbuf_db; 0.00 : ffff8000107d45b4: ldrh w0, [x2] : *dbbuf_db = value; 0.00 : ffff8000107d45b8: str w20, [x2] : mb(); 0.00 : ffff8000107d45bc: dsb sy : if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value)) 0.00 : ffff8000107d45c0: ldr w2, [x3] : nvme_dbbuf_need_event(): : return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old); 0.00 : ffff8000107d45c4: sub w0, w20, w0 : nvme_dbbuf_update_and_check_event(): : if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value)) 0.00 : ffff8000107d45c8: and w0, w0, #0xffff : nvme_dbbuf_need_event(): : return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old); 0.00 : ffff8000107d45cc: mvn w2, w2 0.00 : ffff8000107d45d0: add w2, w20, w2 : nvme_dbbuf_update_and_check_event(): : if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value)) 0.00 : ffff8000107d45d4: cmp w0, w2, uxth 0.00 : ffff8000107d45d8: b.ls ffff8000107d45f4 // b.plast : nvme_ring_cq_doorbell(): : writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 0.86 : ffff8000107d45dc: dmb oshst 3.03 : ffff8000107d45e0: ldr x0, [x1] 0.05 : ffff8000107d45e4: ldr x2, [x1, #96] 0.01 : ffff8000107d45e8: ldr w0, [x0, #392] 0.00 : ffff8000107d45ec: add x0, x2, x0, lsl #2 : __raw_writel(): : } : : #define __raw_writel __raw_writel : static __always_inline void __raw_writel(u32 val, volatile void __iomem *addr) : { : asm volatile("str %w0, [%1]" : : "rZ" (val), "r" (addr)); 0.02 : ffff8000107d45f0: str w20, [x0] : nvme_irq(): : nvme_process_cq(nvmeq, &start, &end, -1); : wmb(); 0.10 : ffff8000107d45f4: dsb st : if (start != end) { : nvme_complete_cqes(nvmeq, start, end); : return IRQ_HANDLED; : } : : return ret; 15.07 : ffff8000107d45f8: mov w0, #0x0 // #0 : if (start != end) { 0.00 : ffff8000107d45fc: cmp w20, w19 0.00 : ffff8000107d4600: b.eq ffff8000107d46e8 // b.none 0.00 : ffff8000107d4604: stp x21, x22, [x29, #32] 0.01 : ffff8000107d4608: mov x22, x1 0.80 : ffff8000107d460c: stp x23, x24, [x29, #48] : nvme_handle_cqe(): : dev_warn(nvmeq->dev->ctrl.device, 0.00 : ffff8000107d4610: adrp x24, ffff800011280000 0.07 : ffff8000107d4614: stp x25, x26, [x29, #64] 0.00 : ffff8000107d4618: add x24, x24, #0x888 : nvme_queue_tagset(): : return nvmeq->dev->tagset.tags[nvmeq->qid - 1]; 0.00 : ffff8000107d461c: mov x23, #0xfffffffffffffff8 // #-8 0.03 : ffff8000107d4620: ldrh w1, [x1, #104] 0.00 : ffff8000107d4624: b ffff8000107d4674 : return nvmeq->dev->admin_tagset.tags[0]; 0.00 : ffff8000107d4628: ldr x0, [x0, #272] 0.00 : ffff8000107d462c: ldr x0, [x0] : nvme_handle_cqe(): : req = blk_mq_tag_to_rq(nvme_queue_tagset(nvmeq), cqe->command_id); 10.00 : ffff8000107d4630: ldrh w1, [x21, #12] 0.00 : ffff8000107d4634: and w1, w1, #0xffff 0.00 : ffff8000107d4638: bl ffff800010463dd8 : trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail); 3.89 : ffff8000107d463c: ldrh w1, [x21, #8] : nvme_end_request(req, cqe->status, cqe->result); 0.01 : ffff8000107d4640: ldrh w1, [x21, #14] 0.00 : ffff8000107d4644: ldr x3, [x26, x25] : nvme_end_request(): : union nvme_result result) : { : struct nvme_request *rq = nvme_req(req); : : rq->status = le16_to_cpu(status) >> 1; : rq->result = result; 0.00 : ffff8000107d4648: str x3, [x0, #288] : rq->status = le16_to_cpu(status) >> 1; 0.00 : ffff8000107d464c: ubfx x1, x1, #1, #15 1.39 : ffff8000107d4650: strh w1, [x0, #298] : /* inject error when permitted by fault injection framework */ : nvme_should_fail(req); : blk_mq_complete_request(req); 0.00 : ffff8000107d4654: bl ffff8000104642f0 : nvme_complete_cqes(): : if (++start == nvmeq->q_depth) 0.33 : ffff8000107d4658: add w19, w19, #0x1 0.02 : ffff8000107d465c: ldrh w1, [x22, #104] 1.55 : ffff8000107d4660: and w19, w19, #0xffff : start = 0; 0.00 : ffff8000107d4664: cmp w19, w1 0.00 : ffff8000107d4668: csel w19, w19, wzr, ne // ne = any : while (start != end) { 10.98 : ffff8000107d466c: cmp w20, w19 0.00 : ffff8000107d4670: b.eq ffff8000107d46d8 // b.none : nvme_handle_cqe(): : volatile struct nvme_completion *cqe = &nvmeq->cqes[idx]; 0.22 : ffff8000107d4674: ldr x26, [x22, #72] 0.07 : ffff8000107d4678: ubfiz x25, x19, #4, #16 2.23 : ffff8000107d467c: ldr x0, [x22] 0.70 : ffff8000107d4680: add x21, x26, x25 : if (unlikely(cqe->command_id >= nvmeq->q_depth)) { 17.60 : ffff8000107d4684: ldrh w2, [x21, #12] 0.01 : ffff8000107d4688: cmp w1, w2, uxth 0.00 : ffff8000107d468c: b.ls ffff8000107d46f4 // b.plast : if (unlikely(nvme_is_aen_req(nvmeq->qid, cqe->command_id))) { 2.81 : ffff8000107d4690: ldrh w1, [x21, #12] 0.28 : ffff8000107d4694: ldrh w2, [x22, #114] 0.00 : ffff8000107d4698: and w1, w1, #0xffff : nvme_is_aen_req(): : put_device(ctrl->device); : } : : static inline bool nvme_is_aen_req(u16 qid, __u16 command_id) : { : return !qid && command_id >= NVME_AQ_BLK_MQ_DEPTH; 0.00 : ffff8000107d469c: cmp w2, #0x0 : nvme_queue_tagset(): : return nvmeq->dev->tagset.tags[nvmeq->qid - 1]; 0.00 : ffff8000107d46a0: add x3, x23, w2, uxtw #3 : nvme_handle_cqe(): : if (unlikely(nvme_is_aen_req(nvmeq->qid, cqe->command_id))) { 0.01 : ffff8000107d46a4: ccmp w1, #0x1e, #0x0, eq // eq = none 0.00 : ffff8000107d46a8: b.hi ffff8000107d4714 // b.pmore : nvme_queue_tagset(): : if (!nvmeq->qid) 0.30 : ffff8000107d46ac: cbz w2, ffff8000107d4628 : return nvmeq->dev->tagset.tags[nvmeq->qid - 1]; 0.10 : ffff8000107d46b0: ldr x0, [x0, #112] 6.86 : ffff8000107d46b4: ldr x0, [x0, x3] 0.01 : ffff8000107d46b8: b ffff8000107d4630 : nvme_update_cq_head(): : nvmeq->cq_phase = !nvmeq->cq_phase; 0.00 : ffff8000107d46bc: cmp w3, #0x0 : nvmeq->cq_head = 0; 0.00 : ffff8000107d46c0: strh wzr, [x1, #112] : nvmeq->cq_phase = !nvmeq->cq_phase; 0.00 : ffff8000107d46c4: cset w3, eq // eq = none 0.00 : ffff8000107d46c8: mov w20, #0x0 // #0 0.00 : ffff8000107d46cc: strb w3, [x1, #116] 0.00 : ffff8000107d46d0: and w4, w3, #0xffff 0.00 : ffff8000107d46d4: b ffff8000107d454c : nvme_irq(): : return IRQ_HANDLED; 0.29 : ffff8000107d46d8: ldp x21, x22, [x29, #32] 0.00 : ffff8000107d46dc: mov w0, #0x1 // #1 0.62 : ffff8000107d46e0: ldp x23, x24, [x29, #48] 0.06 : ffff8000107d46e4: ldp x25, x26, [x29, #64] : } 0.01 : ffff8000107d46e8: ldp x19, x20, [sp, #16] 0.02 : ffff8000107d46ec: ldp x29, x30, [sp], #80 0.00 : ffff8000107d46f0: ret : nvme_handle_cqe(): : dev_warn(nvmeq->dev->ctrl.device, 0.00 : ffff8000107d46f4: ldr x0, [x0, #1448] 0.00 : ffff8000107d46f8: mov x1, x24 0.00 : ffff8000107d46fc: ldrh w2, [x21, #12] 0.00 : ffff8000107d4700: ldrh w3, [x21, #10] 0.00 : ffff8000107d4704: and w2, w2, #0xffff 0.00 : ffff8000107d4708: and w3, w3, #0xffff 0.00 : ffff8000107d470c: bl ffff80001072f0d8 <_dev_warn> 0.00 : ffff8000107d4710: b ffff8000107d4658 : nvme_complete_async_event(&nvmeq->dev->ctrl, 0.00 : ffff8000107d4714: ldrh w1, [x21, #14] 0.00 : ffff8000107d4718: mov x2, x21 0.00 : ffff8000107d471c: add x0, x0, #0x200 0.00 : ffff8000107d4720: bl ffff8000107d0428 0.00 : ffff8000107d4724: b ffff8000107d4658 Percent | Source code & Disassembly of vmlinux for cycles (70202 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff8000104b1058 : : sbitmap_queue_clear(): : } : EXPORT_SYMBOL_GPL(sbitmap_queue_wake_up); : : void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, : unsigned int cpu) : { 0.40 : ffff8000104b1058: stp x29, x30, [sp, #-48]! 0.00 : ffff8000104b105c: mov x29, sp 0.01 : ffff8000104b1060: stp x19, x20, [sp, #16] 0.00 : ffff8000104b1064: mov x19, x0 0.01 : ffff8000104b1068: str x21, [sp, #32] 0.00 : ffff8000104b106c: mov w20, w1 0.00 : ffff8000104b1070: mov w21, w2 : * and its pair is the memory barrier implied in __sbitmap_get_word. : * : * One invariant is that the clear bit has to be zero when the bit : * is in use. : */ : smp_mb__before_atomic(); 0.07 : ffff8000104b1074: dmb ish : sbitmap_deferred_clear_bit(&sbq->sb, nr); 8.01 : ffff8000104b1078: ldr w4, [x0, #4] : sbitmap_deferred_clear_bit(): : */ : static inline void sbitmap_deferred_clear_bit(struct sbitmap *sb, unsigned int bitnr) : { : unsigned long *addr = &sb->map[SB_NR_TO_INDEX(sb, bitnr)].cleared; : : set_bit(SB_NR_TO_BIT(sb, bitnr), addr); 0.00 : ffff8000104b107c: mov w2, #0xffffffff // #-1 : unsigned long *addr = &sb->map[SB_NR_TO_INDEX(sb, bitnr)].cleared; 0.02 : ffff8000104b1080: ldr x0, [x0, #16] 0.00 : ffff8000104b1084: mov w5, #0xc0 // #192 : set_bit(): : */ : : static inline void set_bit(unsigned int nr, volatile unsigned long *p) : { : p += BIT_WORD(nr); : atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff8000104b1088: mov x1, #0x1 // #1 : sbitmap_deferred_clear_bit(): 0.00 : ffff8000104b108c: lsr w3, w20, w4 : set_bit(SB_NR_TO_BIT(sb, bitnr), addr); 0.00 : ffff8000104b1090: lsl w2, w2, w4 0.00 : ffff8000104b1094: bic w2, w20, w2 : unsigned long *addr = &sb->map[SB_NR_TO_INDEX(sb, bitnr)].cleared; 0.00 : ffff8000104b1098: umaddl x3, w3, w5, x0 : set_bit(): : p += BIT_WORD(nr); 0.00 : ffff8000104b109c: lsr w0, w2, #6 : atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff8000104b10a0: lsl x1, x1, x2 : sbitmap_deferred_clear_bit(): 0.00 : ffff8000104b10a4: add x2, x3, #0x80 : set_bit(): : p += BIT_WORD(nr); 0.01 : ffff8000104b10a8: add x0, x2, x0, lsl #3 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000104b10ac: b ffff8000104b1108 0.45 : ffff8000104b10b0: b ffff8000104b1108 : __lse_atomic64_or(): : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) 0.00 : ffff8000104b10b4: stset x1, [x0] : sbitmap_queue_clear(): : * Pairs with the memory barrier in set_current_state() to ensure the : * proper ordering of clear_bit_unlock()/waitqueue_active() in the waker : * and test_and_set_bit_lock()/prepare_to_wait()/finish_wait() in the : * waiter. See the comment on waitqueue_active(). : */ : smp_mb__after_atomic(); 1.77 : ffff8000104b10b8: dmb ish 88.00 : ffff8000104b10bc: nop : sbitmap_queue_wake_up(): : while (__sbq_wake_up(sbq)) 0.00 : ffff8000104b10c0: mov x0, x19 0.00 : ffff8000104b10c4: bl ffff8000104b0ee0 <__sbq_wake_up> 0.00 : ffff8000104b10c8: tst w0, #0xff 0.00 : ffff8000104b10cc: b.ne ffff8000104b10c0 // b.any : sbitmap_queue_clear(): : sbitmap_queue_wake_up(sbq); : : if (likely(!sbq->round_robin && nr < sbq->sb.depth)) 0.00 : ffff8000104b10d0: ldrb w0, [x19, #52] 0.00 : ffff8000104b10d4: cbnz w0, ffff8000104b10f8 0.07 : ffff8000104b10d8: ldr w0, [x19] 0.00 : ffff8000104b10dc: cmp w0, w20 0.12 : ffff8000104b10e0: b.ls ffff8000104b10f8 // b.plast : *per_cpu_ptr(sbq->alloc_hint, cpu) = nr; 0.34 : ffff8000104b10e4: adrp x1, ffff800011909000 0.00 : ffff8000104b10e8: add x1, x1, #0x928 0.00 : ffff8000104b10ec: ldr x0, [x19, #24] 0.01 : ffff8000104b10f0: ldr x1, [x1, w21, uxtw #3] 0.16 : ffff8000104b10f4: str w20, [x1, x0] : } 0.30 : ffff8000104b10f8: ldp x19, x20, [sp, #16] 0.09 : ffff8000104b10fc: ldr x21, [sp, #32] 0.15 : ffff8000104b1100: ldp x29, x30, [sp], #48 0.00 : ffff8000104b1104: ret : __ll_sc_atomic64_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000104b1108: b ffff8000104b153c <__sbitmap_queue_get+0x2a4> 0.00 : ffff8000104b110c: b ffff8000104b10b8 Percent | Source code & Disassembly of vmlinux for cycles (65956 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010120a70 : : enqueue_task_fair(): : * increased. Here we update the fair scheduling stats and : * then put the task into the rbtree: : */ : static void : enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) : { 0.09 : ffff800010120a70: stp x29, x30, [sp, #-128]! 0.00 : ffff800010120a74: mov x29, sp 0.22 : ffff800010120a78: stp x19, x20, [sp, #16] : struct cfs_rq *cfs_rq; : struct sched_entity *se = &p->se; 0.00 : ffff800010120a7c: add x20, x1, #0xc0 : { 0.00 : ffff800010120a80: stp x21, x22, [sp, #32] 0.00 : ffff800010120a84: mov w21, w2 0.22 : ffff800010120a88: stp x23, x24, [sp, #48] 0.00 : ffff800010120a8c: mov x24, x0 : struct sched_entity *se = &p->se; 0.00 : ffff800010120a90: mov x19, x20 : util_est_enqueue(): : enqueued = cfs_rq->avg.util_est.enqueued; 0.01 : ffff800010120a94: ldr w3, [x0, #312] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.16 : ffff800010120a98: ldr x0, [x1, #440] : idle_policy(): : */ : #define RUNTIME_INF ((u64)~0ULL) : : static inline int idle_policy(int policy) : { : return policy == SCHED_IDLE; 0.03 : ffff800010120a9c: ldr w2, [x1, #728] 0.00 : ffff800010120aa0: cmp w2, #0x5 : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff800010120aa4: lsr x2, x0, #32 : idle_policy(): 0.00 : ffff800010120aa8: cset w23, eq // eq = none : _task_util_est(): 0.19 : ffff800010120aac: cmp w0, w2 0.00 : ffff800010120ab0: csel w0, w0, w2, hi // hi = pmore 0.00 : ffff800010120ab4: orr w0, w0, #0x1 : util_est_enqueue(): : enqueued += _task_util_est(p); 0.00 : ffff800010120ab8: add w0, w0, w3 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010120abc: str w0, [x24, #312] : enqueue_task_fair(): : /* : * If in_iowait is set, the code below may not trigger any cpufreq : * utilization updates, so do it here explicitly with the IOWAIT flag : * passed. : */ : if (p->in_iowait) 0.18 : ffff800010120ac0: ldrb w0, [x1, #1064] 0.00 : ffff800010120ac4: tbz w0, #1, ffff800010120af8 : cpufreq_update_util(): : */ : static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) : { : struct update_util_data *data; : : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff800010120ac8: ldrsw x2, [x24, #2568] 0.00 : ffff800010120acc: adrp x1, ffff800011909000 0.00 : ffff800010120ad0: add x1, x1, #0x928 0.00 : ffff800010120ad4: adrp x0, ffff80001151f000 0.00 : ffff800010120ad8: add x0, x0, #0x158 0.00 : ffff800010120adc: ldr x1, [x1, x2, lsl #3] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120ae0: ldr x0, [x0, x1] : cpufreq_update_util(): : cpu_of(rq))); : if (data) 0.00 : ffff800010120ae4: cbz x0, ffff800010120af8 : data->func(data, rq_clock(rq), flags); 0.00 : ffff800010120ae8: ldr x3, [x0] 0.00 : ffff800010120aec: mov w2, #0x1 // #1 0.00 : ffff800010120af0: ldr x1, [x24, #2400] 0.00 : ffff800010120af4: blr x3 0.03 : ffff800010120af8: and w22, w21, #0x1 0.18 : ffff800010120afc: str w22, [x29, #112] : enqueue_task_fair(): : cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); : : for_each_sched_entity(se) { 0.00 : ffff800010120b00: cbz x20, ffff8000101210c8 0.00 : ffff800010120b04: stp x25, x26, [x29, #64] 0.12 : ffff800010120b08: stp x27, x28, [x29, #80] : if (se->on_rq) 0.05 : ffff800010120b0c: ldr w0, [x20, #64] 0.00 : ffff800010120b10: cbnz w0, ffff80001012113c : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 13.82 : ffff800010120b14: adrp x25, ffff800011afb000 0.00 : ffff800010120b18: add x25, x25, #0xd80 : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff800010120b1c: adrp x27, ffff80001151f000 0.00 : ffff800010120b20: add x0, x27, #0x158 0.30 : ffff800010120b24: str x0, [x29, #96] : enqueue_task_fair(): : break; : cfs_rq = cfs_rq_of(se); 0.46 : ffff800010120b28: ldr x20, [x19, #128] : enqueue_entity(): : bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED); 0.00 : ffff800010120b2c: mov w2, #0x41 // #65 0.00 : ffff800010120b30: and w2, w21, w2 0.00 : ffff800010120b34: cmp w2, #0x1 1.78 : ffff800010120b38: cset w28, ne // ne = any : bool curr = cfs_rq->curr == se; 0.01 : ffff800010120b3c: ldr x21, [x20, #72] : if (renorm && curr) 0.00 : ffff800010120b40: cmp w28, #0x0 0.00 : ffff800010120b44: ccmp x21, x19, #0x0, ne // ne = any 0.00 : ffff800010120b48: b.ne ffff800010120f54 // b.any : se->vruntime += cfs_rq->min_vruntime; 0.00 : ffff800010120b4c: ldr x1, [x19, #88] : update_curr(cfs_rq); 0.00 : ffff800010120b50: mov x0, x20 : se->vruntime += cfs_rq->min_vruntime; 0.00 : ffff800010120b54: ldr x2, [x20, #48] 0.00 : ffff800010120b58: add x1, x1, x2 0.00 : ffff800010120b5c: str x1, [x19, #88] : update_curr(cfs_rq); 0.00 : ffff800010120b60: bl ffff80001011fab8 : cfs_rq_clock_pelt(): : return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; : } : #else : static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) : { : return rq_clock_pelt(rq_of(cfs_rq)); 0.60 : ffff800010120b64: ldr x0, [x20, #304] : update_load_avg(): : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 0.44 : ffff800010120b68: ldr x1, [x19, #192] : rq_clock_pelt(): : return rq->clock_pelt - rq->lost_idle_time; 0.65 : ffff800010120b6c: ldr x3, [x0, #2440] 0.93 : ffff800010120b70: ldr x0, [x0, #2448] 0.04 : ffff800010120b74: sub x27, x3, x0 : update_load_avg(): 0.00 : ffff800010120b78: cbz x1, ffff800010120b8c : __update_load_avg_se(now, cfs_rq, se); 0.33 : ffff800010120b7c: mov x2, x19 0.00 : ffff800010120b80: mov x1, x20 0.00 : ffff800010120b84: mov x0, x27 0.00 : ffff800010120b88: bl ffff800010134868 <__update_load_avg_se> : update_cfs_rq_load_avg(): : if (cfs_rq->removed.nr) { 0.70 : ffff800010120b8c: ldr w28, [x20, #196] 0.00 : ffff800010120b90: cbz w28, ffff800010120c44 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010120b94: ldr w1, [x20, #156] 0.00 : ffff800010120b98: mov w3, #0xb67e // #46718 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010120b9c: add x28, x20, #0xc0 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010120ba0: add w1, w1, w3 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010120ba4: mov x0, x28 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010120ba8: str w1, [x29, #104] : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010120bac: bl ffff800010cd80f0 <_raw_spin_lock> : cfs_rq->removed.nr = 0; 0.00 : ffff800010120bb0: str wzr, [x20, #196] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff800010120bb4: ldp x5, x26, [x20, #200] 0.00 : ffff800010120bb8: stp xzr, xzr, [x20, #200] : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010120bbc: ldr x9, [x20, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff800010120bc0: mov x0, x28 : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010120bc4: str xzr, [x20, #216] 0.00 : ffff800010120bc8: stp x5, x9, [x29, #112] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff800010120bcc: bl ffff800010cd7c38 <_raw_spin_unlock> : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120bd0: ldp x5, x9, [x29, #112] : cfs_rq->propagate = 1; 0.00 : ffff800010120bd4: mov x6, #0x1 // #1 : __read_once_size(): 0.00 : ffff800010120bd8: ldr x7, [x20, #160] : update_cfs_rq_load_avg(): : decayed = 1; 0.00 : ffff800010120bdc: mov w28, w6 : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120be0: ldr x4, [x20, #272] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff800010120be4: sub x2, x7, x5 : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010120be8: ldr w1, [x29, #104] : sub_positive(&sa->load_avg, r); 0.00 : ffff800010120bec: cmp x7, x2 : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010120bf0: str x6, [x20, #264] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff800010120bf4: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120bf8: str x2, [x20, #160] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120bfc: sub x2, x4, x9 : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010120c00: mov w0, w1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120c04: ldr x4, [x20, #136] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120c08: str x2, [x20, #272] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010120c0c: msub x0, x0, x5, x4 0.00 : ffff800010120c10: cmp x4, x0 0.00 : ffff800010120c14: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120c18: str x0, [x20, #136] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120c1c: ldr x2, [x20, #176] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_avg, r); 0.00 : ffff800010120c20: sub x0, x2, x26 0.00 : ffff800010120c24: cmp x2, x0 0.00 : ffff800010120c28: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120c2c: str x0, [x20, #176] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120c30: ldr w2, [x20, #152] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_sum, r * divider); 0.00 : ffff800010120c34: msub w0, w1, w26, w2 0.00 : ffff800010120c38: cmp w2, w0 0.00 : ffff800010120c3c: csel w0, w0, wzr, cs // cs = hs, nlast : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010120c40: str w0, [x20, #152] : update_cfs_rq_load_avg(): : decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 0.25 : ffff800010120c44: mov x1, x20 0.00 : ffff800010120c48: mov x0, x27 0.00 : ffff800010120c4c: bl ffff800010134a68 <__update_load_avg_cfs_rq> 1.12 : ffff800010120c50: orr w0, w0, w28 : propagate_entity_load_avg(): : if (entity_is_task(se)) 0.09 : ffff800010120c54: ldr x1, [x19, #136] 0.00 : ffff800010120c58: cbz x1, ffff800010120f00 : if (!gcfs_rq->propagate) 0.19 : ffff800010120c5c: ldr x2, [x1, #264] 0.03 : ffff800010120c60: cbz x2, ffff800010120f00 : gcfs_rq->propagate = 0; 0.00 : ffff800010120c64: str xzr, [x1, #264] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010120c68: mov x3, #0x1 // #1 : propagate_entity_load_avg(): : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 0.00 : ffff800010120c6c: ldr x2, [x1, #272] : cfs_rq = cfs_rq_of(se); 0.00 : ffff800010120c70: ldr x4, [x19, #128] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120c74: ldr x0, [x4, #272] 0.00 : ffff800010120c78: add x0, x0, x2 0.00 : ffff800010120c7c: stp x3, x0, [x4, #264] : propagate_entity_load_avg(): : update_tg_cfs_util(cfs_rq, se, gcfs_rq); 0.00 : ffff800010120c80: ldr x2, [x1, #176] : update_tg_cfs_util(): : long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 0.00 : ffff800010120c84: ldr x0, [x19, #240] 0.00 : ffff800010120c88: sub x0, x2, x0 : if (!delta) 0.00 : ffff800010120c8c: cmp x0, #0x0 0.00 : ffff800010120c90: b.eq ffff800010120cc4 // b.none : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010120c94: mov w3, #0xba7e // #47742 : se->avg.util_avg = gcfs_rq->avg.util_avg; 0.00 : ffff800010120c98: str x2, [x19, #240] : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010120c9c: mul w2, w2, w3 0.00 : ffff800010120ca0: str w2, [x19, #216] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120ca4: ldr x2, [x4, #176] : update_tg_cfs_util(): : add_positive(&cfs_rq->avg.util_avg, delta); 0.00 : ffff800010120ca8: add x0, x0, x2 0.00 : ffff800010120cac: ccmp x2, x0, #0x2, lt // lt = tstop 0.00 : ffff800010120cb0: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120cb4: str x0, [x4, #176] : update_tg_cfs_util(): : cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010120cb8: ldr x0, [x4, #176] 0.00 : ffff800010120cbc: mul w3, w3, w0 0.00 : ffff800010120cc0: str w3, [x4, #152] : update_tg_cfs_runnable(): : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 0.00 : ffff800010120cc4: ldr x3, [x1, #272] : if (!runnable_sum) 0.00 : ffff800010120cc8: cmp x3, #0x0 0.00 : ffff800010120ccc: b.eq ffff800010120de8 // b.none : gcfs_rq->prop_runnable_sum = 0; 0.00 : ffff800010120cd0: str xzr, [x1, #272] : if (runnable_sum >= 0) { 0.00 : ffff800010120cd4: b.lt ffff800010121470 // b.tstop : runnable_sum += se->avg.load_sum; 0.00 : ffff800010120cd8: ldr x0, [x19, #200] : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff800010120cdc: mov x1, #0xba7e // #47742 : runnable_sum += se->avg.load_sum; 0.00 : ffff800010120ce0: add x3, x3, x0 : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff800010120ce4: cmp x3, x1 0.00 : ffff800010120ce8: csel x3, x3, x1, le : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff800010120cec: ldr w1, [x19, #216] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010120cf0: ldr x2, [x19] : update_tg_cfs_runnable(): : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff800010120cf4: ldr x5, [x19, #224] : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff800010120cf8: lsr w1, w1, #10 : runnable_sum = max(runnable_sum, running_sum); 0.00 : ffff800010120cfc: cmp x1, x3 : se_weight(): 0.00 : ffff800010120d00: lsr x2, x2, #10 : update_tg_cfs_runnable(): 0.00 : ffff800010120d04: csel x3, x1, x3, ge // ge = tcont : se->avg.load_sum = runnable_sum; 0.00 : ffff800010120d08: str x3, [x19, #200] : load_sum = (s64)se_weight(se) * runnable_sum; 0.00 : ffff800010120d0c: mul x1, x2, x3 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 0.00 : ffff800010120d10: msub x2, x2, x0, x1 : div_s64_rem(): : * Return: sets ``*remainder``, then returns dividend / divisor : */ : static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) : { : *remainder = dividend % divisor; : return dividend / divisor; 0.00 : ffff800010120d14: lsr x0, x1, #1 0.00 : ffff800010120d18: mov x1, #0x5041 // #20545 0.00 : ffff800010120d1c: movk x1, #0xafb, lsl #16 0.00 : ffff800010120d20: movk x1, #0x400, lsl #32 0.00 : ffff800010120d24: movk x1, #0xafb5, lsl #48 0.00 : ffff800010120d28: umulh x0, x0, x1 0.00 : ffff800010120d2c: lsr x0, x0, #14 : update_tg_cfs_runnable(): : se->avg.load_avg = load_avg; 0.00 : ffff800010120d30: str x0, [x19, #224] : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff800010120d34: sub x0, x0, x5 : div_s64_rem(): 0.00 : ffff800010120d38: mov x5, #0x5041 // #20545 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120d3c: ldr x1, [x4, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_avg, delta_avg); 0.00 : ffff800010120d40: cmp x0, #0x0 : div_s64_rem(): 0.00 : ffff800010120d44: movk x5, #0xafb, lsl #16 : update_tg_cfs_runnable(): 0.00 : ffff800010120d48: add x0, x0, x1 : div_s64_rem(): 0.00 : ffff800010120d4c: movk x5, #0x400, lsl #32 : update_tg_cfs_runnable(): 0.00 : ffff800010120d50: ccmp x1, x0, #0x2, lt // lt = tstop : div_s64_rem(): 0.00 : ffff800010120d54: movk x5, #0xafb5, lsl #48 : update_tg_cfs_runnable(): 0.00 : ffff800010120d58: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120d5c: str x0, [x4, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_sum, delta_sum); 0.00 : ffff800010120d60: cmp x2, #0x0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120d64: ldr x1, [x4, #136] : update_tg_cfs_runnable(): 0.00 : ffff800010120d68: add x0, x2, x1 0.00 : ffff800010120d6c: ccmp x1, x0, #0x2, lt // lt = tstop 0.00 : ffff800010120d70: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120d74: str x0, [x4, #136] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff800010120d78: ldr x1, [x19, #16] : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff800010120d7c: ldr w2, [x19, #64] : se_runnable(): 0.00 : ffff800010120d80: lsr x1, x1, #10 : update_tg_cfs_runnable(): : runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 0.00 : ffff800010120d84: mul x1, x1, x3 : div_s64_rem(): 0.00 : ffff800010120d88: lsr x0, x1, #1 0.00 : ffff800010120d8c: umulh x0, x0, x5 0.00 : ffff800010120d90: lsr x0, x0, #14 : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff800010120d94: cbz w2, ffff800010120de0 : delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 0.00 : ffff800010120d98: ldr x6, [x19, #232] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120d9c: ldr x7, [x4, #168] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010120da0: ldr x2, [x19] : update_tg_cfs_runnable(): 0.00 : ffff800010120da4: sub x6, x0, x6 : se_weight(se) * se->avg.runnable_load_sum; 0.00 : ffff800010120da8: ldr x8, [x19, #208] : add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 0.00 : ffff800010120dac: add x5, x6, x7 0.00 : ffff800010120db0: cmp x6, #0x0 0.00 : ffff800010120db4: ccmp x7, x5, #0x2, lt // lt = tstop : se_weight(): 0.00 : ffff800010120db8: lsr x2, x2, #10 : update_tg_cfs_runnable(): 0.00 : ffff800010120dbc: csel x5, x5, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120dc0: str x5, [x4, #168] : update_tg_cfs_runnable(): : delta_sum = runnable_load_sum - 0.00 : ffff800010120dc4: msub x1, x2, x8, x1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120dc8: ldr x2, [x4, #144] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 0.00 : ffff800010120dcc: cmp x1, #0x0 0.00 : ffff800010120dd0: add x1, x1, x2 0.00 : ffff800010120dd4: ccmp x2, x1, #0x2, lt // lt = tstop 0.00 : ffff800010120dd8: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120ddc: str x1, [x4, #144] : update_tg_cfs_runnable(): : se->avg.runnable_load_sum = runnable_sum; 0.00 : ffff800010120de0: str x3, [x19, #208] : se->avg.runnable_load_avg = runnable_load_avg; 0.00 : ffff800010120de4: str x0, [x19, #232] : update_load_avg(): : if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 0.00 : ffff800010120de8: ldr x0, [x19, #192] 0.00 : ffff800010120dec: cbnz x0, ffff800010120f0c : attach_entity_load_avg(cfs_rq, se); 0.00 : ffff800010120df0: mov x1, x19 0.00 : ffff800010120df4: mov x0, x20 0.00 : ffff800010120df8: bl ffff80001011c660 : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff800010120dfc: ldr x1, [x20, #336] 0.00 : ffff800010120e00: cmp x1, x25 0.00 : ffff800010120e04: b.eq ffff800010120e38 // b.none : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 0.00 : ffff800010120e08: ldr x0, [x20, #160] 0.00 : ffff800010120e0c: ldr x2, [x20, #256] : if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 0.00 : ffff800010120e10: subs x0, x0, x2 0.00 : ffff800010120e14: cneg x3, x0, mi // mi = first 0.00 : ffff800010120e18: cmp x3, x2, lsr #6 0.00 : ffff800010120e1c: b.ls ffff800010120e38 // b.plast : atomic_long_add(delta, &cfs_rq->tg->load_avg); 0.00 : ffff800010120e20: add x2, x1, #0x140 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010120e24: b ffff80001012112c 0.00 : ffff800010120e28: b ffff80001012112c : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff800010120e2c: stadd x0, [x2] : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff800010120e30: ldr x0, [x20, #160] 0.00 : ffff800010120e34: str x0, [x20, #256] : enqueue_entity(): : update_cfs_group(se); 0.04 : ffff800010120e38: mov x0, x19 0.00 : ffff800010120e3c: bl ffff80001011fd78 : enqueue_runnable_load_avg(): : cfs_rq->runnable_weight += se->runnable_weight; 0.12 : ffff800010120e40: ldr x1, [x19, #16] 6.31 : ffff800010120e44: ldr x0, [x20, #16] : cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg; 0.67 : ffff800010120e48: ldr x3, [x20, #168] : cfs_rq->runnable_weight += se->runnable_weight; 0.10 : ffff800010120e4c: add x0, x0, x1 0.24 : ffff800010120e50: str x0, [x20, #16] : cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum; 0.10 : ffff800010120e54: ldr x4, [x20, #144] : enqueue_entity(): : account_entity_enqueue(cfs_rq, se); 0.01 : ffff800010120e58: mov x1, x19 : enqueue_runnable_load_avg(): : cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg; 0.31 : ffff800010120e5c: ldr x2, [x19, #232] : enqueue_entity(): : account_entity_enqueue(cfs_rq, se); 0.00 : ffff800010120e60: mov x0, x20 : enqueue_runnable_load_avg(): : cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg; 0.00 : ffff800010120e64: add x3, x3, x2 2.29 : ffff800010120e68: str x3, [x20, #168] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.22 : ffff800010120e6c: ldr x2, [x19, #16] : enqueue_runnable_load_avg(): : cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum; 0.02 : ffff800010120e70: ldr x3, [x19, #208] : se_runnable(): 0.00 : ffff800010120e74: lsr x2, x2, #10 : enqueue_runnable_load_avg(): 0.01 : ffff800010120e78: madd x2, x2, x3, x4 0.26 : ffff800010120e7c: str x2, [x20, #144] : enqueue_entity(): : account_entity_enqueue(cfs_rq, se); 0.00 : ffff800010120e80: bl ffff80001011c540 : if (flags & ENQUEUE_WAKEUP) 0.16 : ffff800010120e84: cbz w22, ffff800010120eac : place_entity(): : unsigned long thresh = sysctl_sched_latency; 0.27 : ffff800010120e88: adrp x0, ffff800011920000 : u64 vruntime = cfs_rq->min_vruntime; 0.20 : ffff800010120e8c: ldr x2, [x20, #48] : se->vruntime = max_vruntime(se->vruntime, vruntime); 0.17 : ffff800010120e90: ldr x1, [x19, #88] : unsigned long thresh = sysctl_sched_latency; 0.12 : ffff800010120e94: ldr w0, [x0, #2272] : vruntime -= thresh; 0.01 : ffff800010120e98: sub x0, x2, x0, lsr #1 : max_vruntime(): : s64 delta = (s64)(vruntime - max_vruntime); 0.05 : ffff800010120e9c: sub x2, x0, x1 : if (delta > 0) 0.01 : ffff800010120ea0: cmp x2, #0x0 0.17 : ffff800010120ea4: csel x0, x0, x1, gt : place_entity(): : se->vruntime = max_vruntime(se->vruntime, vruntime); 0.18 : ffff800010120ea8: str x0, [x19, #88] : enqueue_entity(): : if (!curr) 0.00 : ffff800010120eac: cmp x21, x19 0.01 : ffff800010120eb0: b.eq ffff800010120ec0 // b.none : __enqueue_entity(cfs_rq, se); 0.15 : ffff800010120eb4: mov x1, x19 0.00 : ffff800010120eb8: mov x0, x20 0.00 : ffff800010120ebc: bl ffff80001011ce78 <__enqueue_entity> : se->on_rq = 1; 0.21 : ffff800010120ec0: mov w0, #0x1 // #1 0.14 : ffff800010120ec4: str w0, [x19, #64] : if (cfs_rq->nr_running == 1) { 0.62 : ffff800010120ec8: ldr w0, [x20, #24] 0.01 : ffff800010120ecc: cmp w0, #0x1 0.00 : ffff800010120ed0: b.eq ffff800010120f7c // b.none : enqueue_task_fair(): : * post the final h_nr_running increment below. : */ : if (cfs_rq_throttled(cfs_rq)) : break; : cfs_rq->h_nr_running++; : cfs_rq->idle_h_nr_running += idle_h_nr_running; 0.22 : ffff800010120ed4: ldp w1, w0, [x20, #28] : cfs_rq->h_nr_running++; 0.00 : ffff800010120ed8: add w1, w1, #0x1 : cfs_rq->idle_h_nr_running += idle_h_nr_running; 0.01 : ffff800010120edc: add w0, w0, w23 0.95 : ffff800010120ee0: stp w1, w0, [x20, #28] : for_each_sched_entity(se) { 1.16 : ffff800010120ee4: ldr x19, [x19, #120] 0.01 : ffff800010120ee8: cbz x19, ffff800010120fe0 : if (se->on_rq) 0.04 : ffff800010120eec: ldr w0, [x19, #64] : : flags = ENQUEUE_WAKEUP; 0.00 : ffff800010120ef0: mov w21, #0x1 // #1 : if (se->on_rq) 0.00 : ffff800010120ef4: cbnz w0, ffff800010121138 27.90 : ffff800010120ef8: mov w22, w21 0.00 : ffff800010120efc: b ffff800010120b28 : update_load_avg(): : if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 0.85 : ffff800010120f00: ldr x1, [x19, #192] 0.00 : ffff800010120f04: cbz x1, ffff800010120df0 : } else if (decayed) { 0.17 : ffff800010120f08: cbz w0, ffff800010120e38 : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010120f0c: ldr x1, [x20, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff800010120f10: add x0, x1, #0x80 0.00 : ffff800010120f14: cmp x20, x0 0.00 : ffff800010120f18: b.eq ffff800010121070 // b.none : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.10 : ffff800010120f1c: ldr x0, [x20, #336] 0.00 : ffff800010120f20: cmp x0, x25 0.00 : ffff800010120f24: b.eq ffff800010120e38 // b.none : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 0.00 : ffff800010120f28: ldr x1, [x20, #160] 0.03 : ffff800010120f2c: ldr x2, [x20, #256] : if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 0.00 : ffff800010120f30: subs x1, x1, x2 0.00 : ffff800010120f34: cneg x3, x1, mi // mi = first 0.00 : ffff800010120f38: cmp x3, x2, lsr #6 0.00 : ffff800010120f3c: b.ls ffff800010120e38 // b.plast : atomic_long_add(delta, &cfs_rq->tg->load_avg); 0.00 : ffff800010120f40: add x2, x0, #0x140 : arch_static_branch_jump(): 0.00 : ffff800010120f44: b ffff800010121118 0.00 : ffff800010120f48: b ffff800010121118 : __lse_atomic64_add(): 0.00 : ffff800010120f4c: stadd x1, [x2] 0.00 : ffff800010120f50: b ffff800010121120 : enqueue_entity(): : update_curr(cfs_rq); 26.95 : ffff800010120f54: mov x0, x20 0.00 : ffff800010120f58: bl ffff80001011fab8 : if (renorm && !curr) 1.22 : ffff800010120f5c: cmp w28, #0x0 0.00 : ffff800010120f60: ccmp x21, x19, #0x4, ne // ne = any 0.00 : ffff800010120f64: b.eq ffff800010120b64 // b.none : se->vruntime += cfs_rq->min_vruntime; 0.00 : ffff800010120f68: ldr x0, [x19, #88] 0.00 : ffff800010120f6c: ldr x1, [x20, #48] 0.00 : ffff800010120f70: add x0, x0, x1 0.00 : ffff800010120f74: str x0, [x19, #88] 0.00 : ffff800010120f78: b ffff800010120b64 : list_add_leaf_cfs_rq(): : if (cfs_rq->on_list) 0.14 : ffff800010120f7c: ldr w1, [x20, #312] 0.01 : ffff800010120f80: cbnz w1, ffff800010120ed4 : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010120f84: ldr x1, [x20, #304] 0.00 : ffff800010120f88: add x2, x20, #0x140 : if (cfs_rq->tg->parent && 0.00 : ffff800010120f8c: ldr x3, [x20, #336] : int cpu = cpu_of(rq); 0.00 : ffff800010120f90: ldr w4, [x1, #2568] : cfs_rq->on_list = 1; 0.00 : ffff800010120f94: str w0, [x20, #312] : if (cfs_rq->tg->parent && 0.00 : ffff800010120f98: ldr x0, [x3, #360] 0.00 : ffff800010120f9c: cbz x0, ffff8000101214dc : cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 0.00 : ffff800010120fa0: ldr x0, [x0, #248] 0.00 : ffff800010120fa4: ldr x0, [x0, w4, sxtw #3] : if (cfs_rq->tg->parent && 0.00 : ffff800010120fa8: ldr w3, [x0, #312] 0.00 : ffff800010120fac: cbnz w3, ffff8000101210f8 : list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 0.00 : ffff800010120fb0: ldr x0, [x1, #2336] : list_add_rcu(): : * the _rcu list-traversal primitives, such as : * list_for_each_entry_rcu(). : */ : static inline void list_add_rcu(struct list_head *new, struct list_head *head) : { : __list_add_rcu(new, head, head->next); 0.00 : ffff800010120fb4: ldr x3, [x0] : __list_add_rcu(): : new->prev = prev; 0.00 : ffff800010120fb8: stp x3, x0, [x20, #320] : rcu_assign_pointer(list_next_rcu(prev), new); 0.00 : ffff800010120fbc: stlr x2, [x0] : next->prev = new; 0.00 : ffff800010120fc0: str x2, [x3, #8] : list_add_leaf_cfs_rq(): : rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 0.00 : ffff800010120fc4: str x2, [x1, #2336] : enqueue_task_fair(): : cfs_rq->idle_h_nr_running += idle_h_nr_running; 0.00 : ffff800010120fc8: ldp w1, w0, [x20, #28] : cfs_rq->h_nr_running++; 0.00 : ffff800010120fcc: add w1, w1, #0x1 : cfs_rq->idle_h_nr_running += idle_h_nr_running; 0.00 : ffff800010120fd0: add w0, w0, w23 0.00 : ffff800010120fd4: stp w1, w0, [x20, #28] : for_each_sched_entity(se) { 0.00 : ffff800010120fd8: ldr x19, [x19, #120] 0.00 : ffff800010120fdc: cbnz x19, ffff800010120eec : add_nr_running(): : unsigned prev_nr = rq->nr_running; 0.04 : ffff800010120fe0: ldr w0, [x24, #4] : rq->nr_running = prev_nr + count; 0.00 : ffff800010120fe4: add w1, w0, #0x1 0.34 : ffff800010120fe8: str w1, [x24, #4] : if (prev_nr < 2 && rq->nr_running >= 2) { 0.00 : ffff800010120fec: cmp w0, #0x1 0.00 : ffff800010120ff0: b.eq ffff8000101214fc // b.none 0.31 : ffff800010120ff4: ldp x25, x26, [x29, #64] 0.11 : ffff800010120ff8: ldp x27, x28, [x29, #80] 0.15 : ffff800010120ffc: ldr x2, [x24, #2464] : __read_once_size(): : __READ_ONCE_SIZE; 0.16 : ffff800010121000: ldr w0, [x2, #92] : update_overutilized_status(): : if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) { 0.00 : ffff800010121004: cbnz w0, ffff8000101210e4 : cpu_util(): : static inline unsigned long cpu_util(int cpu) : { : struct cfs_rq *cfs_rq; : unsigned int util; : : cfs_rq = &cpu_rq(cpu)->cfs; 0.19 : ffff800010121008: ldrsw x3, [x24, #2568] 0.00 : ffff80001012100c: adrp x1, ffff800011909000 0.00 : ffff800010121010: add x1, x1, #0x928 0.00 : ffff800010121014: adrp x0, ffff800011528000 0.03 : ffff800010121018: add x0, x0, #0xe80 0.00 : ffff80001012101c: ldr x1, [x1, x3, lsl #3] 0.00 : ffff800010121020: add x0, x0, x1 : __read_once_size(): 0.04 : ffff800010121024: ldr x3, [x0, #304] 0.08 : ffff800010121028: ldr w5, [x0, #312] : cpu_util(): : util = READ_ONCE(cfs_rq->avg.util_avg); : : if (sched_feat(UTIL_EST)) : util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); : : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.03 : ffff80001012102c: ldr x1, [x0, #2488] : util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); 0.00 : ffff800010121030: cmp w5, w3 : cpu_overutilized(): : return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); 0.02 : ffff800010121034: ldr x4, [x0, #2480] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff800010121038: csel w0, w3, w5, ls // ls = plast 0.00 : ffff80001012103c: cmp x1, x0 0.00 : ffff800010121040: csel x0, x1, x0, ls // ls = plast : cpu_overutilized(): : return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); 0.00 : ffff800010121044: lsl x3, x4, #10 0.00 : ffff800010121048: add x0, x0, x0, lsl #2 : update_overutilized_status(): : if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) { 0.00 : ffff80001012104c: cmp x3, x0, lsl #8 0.00 : ffff800010121050: b.hi ffff8000101210e4 // b.pmore : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010121054: mov w0, #0x2 // #2 0.00 : ffff800010121058: str w0, [x2, #92] : enqueue_task_fair(): : } 0.00 : ffff80001012105c: ldp x19, x20, [sp, #16] 0.00 : ffff800010121060: ldp x21, x22, [sp, #32] 0.00 : ffff800010121064: ldp x23, x24, [sp, #48] 0.00 : ffff800010121068: ldp x29, x30, [sp], #128 0.00 : ffff80001012106c: ret : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff800010121070: ldrsw x3, [x1, #2568] 0.00 : ffff800010121074: adrp x2, ffff800011909000 0.00 : ffff800010121078: add x2, x2, #0x928 0.00 : ffff80001012107c: ldr x0, [x29, #96] 0.01 : ffff800010121080: ldr x2, [x2, x3, lsl #3] : __read_once_size(): : __READ_ONCE_SIZE; 0.01 : ffff800010121084: ldr x0, [x0, x2] : cpufreq_update_util(): : if (data) 0.00 : ffff800010121088: cbz x0, ffff800010120f1c : data->func(data, rq_clock(rq), flags); 0.00 : ffff80001012108c: ldr x3, [x0] 0.00 : ffff800010121090: mov w2, #0x0 // #0 0.00 : ffff800010121094: ldr x1, [x1, #2400] 0.00 : ffff800010121098: blr x3 0.00 : ffff80001012109c: b ffff800010120f1c : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff8000101210a0: add x0, x0, #0x140 0.00 : ffff8000101210a4: b ffff800010128a34 : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff8000101210a8: ldr x0, [x20, #160] 0.00 : ffff8000101210ac: str x0, [x20, #256] : enqueue_task_fair(): : update_cfs_group(se); 0.00 : ffff8000101210b0: mov x0, x19 0.00 : ffff8000101210b4: bl ffff80001011fd78 : for_each_sched_entity(se) { 0.00 : ffff8000101210b8: ldr x19, [x19, #120] 0.00 : ffff8000101210bc: cbnz x19, ffff800010121158 0.00 : ffff8000101210c0: ldp x25, x26, [x29, #64] 0.00 : ffff8000101210c4: ldp x27, x28, [x29, #80] : add_nr_running(): : unsigned prev_nr = rq->nr_running; 0.00 : ffff8000101210c8: ldr w0, [x24, #4] : rq->nr_running = prev_nr + count; 0.00 : ffff8000101210cc: add w1, w0, #0x1 0.00 : ffff8000101210d0: str w1, [x24, #4] : if (prev_nr < 2 && rq->nr_running >= 2) { 0.00 : ffff8000101210d4: cmp w0, #0x1 0.00 : ffff8000101210d8: b.eq ffff8000101214c8 // b.none : enqueue_task_fair(): : if (flags & ENQUEUE_WAKEUP) 0.00 : ffff8000101210dc: ldr w0, [x29, #112] 0.00 : ffff8000101210e0: cbnz w0, ffff80001012151c : } 1.78 : ffff8000101210e4: ldp x19, x20, [sp, #16] 0.45 : ffff8000101210e8: ldp x21, x22, [sp, #32] 0.03 : ffff8000101210ec: ldp x23, x24, [sp, #48] 0.04 : ffff8000101210f0: ldp x29, x30, [sp], #128 0.00 : ffff8000101210f4: ret : list_add_tail_rcu(): : * list_for_each_entry_rcu(). : */ : static inline void list_add_tail_rcu(struct list_head *new, : struct list_head *head) : { : __list_add_rcu(new, head->prev, head); 0.00 : ffff8000101210f8: ldr x3, [x0, #328] : list_add_leaf_cfs_rq(): : list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 0.00 : ffff8000101210fc: add x4, x0, #0x140 : __list_add_rcu(): : new->prev = prev; 0.00 : ffff800010121100: stp x4, x3, [x20, #320] : rcu_assign_pointer(list_next_rcu(prev), new); 0.00 : ffff800010121104: stlr x2, [x3] : next->prev = new; 0.00 : ffff800010121108: str x2, [x0, #328] : list_add_leaf_cfs_rq(): : rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 0.00 : ffff80001012110c: add x0, x1, #0x910 0.00 : ffff800010121110: str x0, [x1, #2336] 0.00 : ffff800010121114: b ffff800010120ed4 : __ll_sc_atomic64_add(): 0.00 : ffff800010121118: add x0, x0, #0x140 0.00 : ffff80001012111c: b ffff800010128a4c : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff800010121120: ldr x0, [x20, #160] 0.00 : ffff800010121124: str x0, [x20, #256] 0.00 : ffff800010121128: b ffff800010120e38 : __ll_sc_atomic64_add(): 0.00 : ffff80001012112c: add x1, x1, #0x140 0.00 : ffff800010121130: b ffff800010128a64 0.00 : ffff800010121134: b ffff800010121120 : enqueue_task_fair(): 0.00 : ffff800010121138: str w21, [x29, #112] : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff80001012113c: adrp x22, ffff800011afb000 0.00 : ffff800010121140: add x0, x22, #0xd80 : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff800010121144: adrp x27, ffff80001151f000 : update_tg_load_avg(): 0.00 : ffff800010121148: str x0, [x29, #120] : cpufreq_update_util(): 0.00 : ffff80001012114c: add x0, x27, #0x158 0.00 : ffff800010121150: str x0, [x29, #104] 0.00 : ffff800010121154: nop : enqueue_task_fair(): : cfs_rq = cfs_rq_of(se); 0.00 : ffff800010121158: ldr x20, [x19, #128] : cfs_rq->idle_h_nr_running += idle_h_nr_running; 0.00 : ffff80001012115c: ldp w1, w0, [x20, #28] : cfs_rq_clock_pelt(): : return rq_clock_pelt(rq_of(cfs_rq)); 0.00 : ffff800010121160: ldr x2, [x20, #304] : enqueue_task_fair(): : cfs_rq->h_nr_running++; 0.00 : ffff800010121164: add w1, w1, #0x1 : cfs_rq->idle_h_nr_running += idle_h_nr_running; 0.00 : ffff800010121168: add w0, w0, w23 0.00 : ffff80001012116c: stp w1, w0, [x20, #28] : update_load_avg(): : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 0.00 : ffff800010121170: ldr x0, [x19, #192] : rq_clock_pelt(): : return rq->clock_pelt - rq->lost_idle_time; 0.00 : ffff800010121174: ldr x28, [x2, #2440] 0.00 : ffff800010121178: ldr x1, [x2, #2448] 0.00 : ffff80001012117c: sub x28, x28, x1 : update_load_avg(): 0.00 : ffff800010121180: cbz x0, ffff800010121194 : __update_load_avg_se(now, cfs_rq, se); 0.00 : ffff800010121184: mov x2, x19 0.00 : ffff800010121188: mov x1, x20 0.00 : ffff80001012118c: mov x0, x28 0.00 : ffff800010121190: bl ffff800010134868 <__update_load_avg_se> : update_cfs_rq_load_avg(): : if (cfs_rq->removed.nr) { 0.00 : ffff800010121194: ldr w21, [x20, #196] 0.00 : ffff800010121198: cbz w21, ffff80001012123c : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff80001012119c: ldr w1, [x20, #156] 0.00 : ffff8000101211a0: mov w2, #0xb67e // #46718 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff8000101211a4: add x21, x20, #0xc0 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff8000101211a8: add w26, w1, w2 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff8000101211ac: mov x0, x21 0.00 : ffff8000101211b0: bl ffff800010cd80f0 <_raw_spin_lock> : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff8000101211b4: ldr x22, [x20, #216] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff8000101211b8: ldp x25, x27, [x20, #200] : cfs_rq->removed.nr = 0; 0.00 : ffff8000101211bc: str wzr, [x20, #196] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff8000101211c0: stp xzr, xzr, [x20, #200] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff8000101211c4: mov x0, x21 : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff8000101211c8: str xzr, [x20, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff8000101211cc: bl ffff800010cd7c38 <_raw_spin_unlock> : __read_once_size(): 0.00 : ffff8000101211d0: ldr x7, [x20, #160] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff8000101211d4: mov w0, w26 : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff8000101211d8: ldr x3, [x20, #272] : cfs_rq->propagate = 1; 0.00 : ffff8000101211dc: mov x6, #0x1 // #1 : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff8000101211e0: sub x2, x7, x25 : decayed = 1; 0.00 : ffff8000101211e4: mov w21, w6 : sub_positive(&sa->load_avg, r); 0.00 : ffff8000101211e8: cmp x7, x2 : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff8000101211ec: str x6, [x20, #264] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff8000101211f0: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101211f4: str x2, [x20, #160] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff8000101211f8: sub x2, x3, x22 0.00 : ffff8000101211fc: str x2, [x20, #272] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121200: ldr x3, [x20, #136] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010121204: msub x0, x0, x25, x3 0.00 : ffff800010121208: cmp x3, x0 0.00 : ffff80001012120c: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010121210: str x0, [x20, #136] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121214: ldr x2, [x20, #176] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_avg, r); 0.00 : ffff800010121218: sub x0, x2, x27 0.00 : ffff80001012121c: cmp x2, x0 0.00 : ffff800010121220: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010121224: str x0, [x20, #176] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121228: ldr w2, [x20, #152] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_sum, r * divider); 0.00 : ffff80001012122c: msub w0, w26, w27, w2 0.00 : ffff800010121230: cmp w2, w0 0.00 : ffff800010121234: csel w0, w0, wzr, cs // cs = hs, nlast : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010121238: str w0, [x20, #152] : update_cfs_rq_load_avg(): : decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 0.00 : ffff80001012123c: mov x1, x20 0.00 : ffff800010121240: mov x0, x28 0.00 : ffff800010121244: bl ffff800010134a68 <__update_load_avg_cfs_rq> 0.00 : ffff800010121248: orr w0, w0, w21 : propagate_entity_load_avg(): : if (entity_is_task(se)) 0.00 : ffff80001012124c: ldr x1, [x19, #136] 0.00 : ffff800010121250: cbz x1, ffff80001012142c : if (!gcfs_rq->propagate) 0.00 : ffff800010121254: ldr x2, [x1, #264] 0.00 : ffff800010121258: cbz x2, ffff80001012142c : gcfs_rq->propagate = 0; 0.00 : ffff80001012125c: str xzr, [x1, #264] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010121260: mov x3, #0x1 // #1 : propagate_entity_load_avg(): : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 0.00 : ffff800010121264: ldr x2, [x1, #272] : cfs_rq = cfs_rq_of(se); 0.00 : ffff800010121268: ldr x4, [x19, #128] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff80001012126c: ldr x0, [x4, #272] 0.00 : ffff800010121270: add x0, x0, x2 0.00 : ffff800010121274: stp x3, x0, [x4, #264] : propagate_entity_load_avg(): : update_tg_cfs_util(cfs_rq, se, gcfs_rq); 0.00 : ffff800010121278: ldr x2, [x1, #176] : update_tg_cfs_util(): : long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 0.00 : ffff80001012127c: ldr x0, [x19, #240] 0.00 : ffff800010121280: sub x0, x2, x0 : if (!delta) 0.00 : ffff800010121284: cmp x0, #0x0 0.00 : ffff800010121288: b.eq ffff8000101212bc // b.none : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff80001012128c: mov w3, #0xba7e // #47742 : se->avg.util_avg = gcfs_rq->avg.util_avg; 0.00 : ffff800010121290: str x2, [x19, #240] : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010121294: mul w2, w2, w3 0.00 : ffff800010121298: str w2, [x19, #216] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012129c: ldr x2, [x4, #176] : update_tg_cfs_util(): : add_positive(&cfs_rq->avg.util_avg, delta); 0.00 : ffff8000101212a0: add x0, x0, x2 0.00 : ffff8000101212a4: ccmp x2, x0, #0x2, lt // lt = tstop 0.00 : ffff8000101212a8: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101212ac: str x0, [x4, #176] : update_tg_cfs_util(): : cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff8000101212b0: ldr x0, [x4, #176] 0.00 : ffff8000101212b4: mul w3, w3, w0 0.00 : ffff8000101212b8: str w3, [x4, #152] : update_tg_cfs_runnable(): : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 0.00 : ffff8000101212bc: ldr x3, [x1, #272] : if (!runnable_sum) 0.00 : ffff8000101212c0: cmp x3, #0x0 0.00 : ffff8000101212c4: b.eq ffff8000101213e0 // b.none : gcfs_rq->prop_runnable_sum = 0; 0.00 : ffff8000101212c8: str xzr, [x1, #272] : if (runnable_sum >= 0) { 0.00 : ffff8000101212cc: b.lt ffff80001012149c // b.tstop : runnable_sum += se->avg.load_sum; 0.00 : ffff8000101212d0: ldr x0, [x19, #200] : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff8000101212d4: mov x1, #0xba7e // #47742 : runnable_sum += se->avg.load_sum; 0.00 : ffff8000101212d8: add x3, x3, x0 : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff8000101212dc: cmp x3, x1 0.00 : ffff8000101212e0: csel x3, x3, x1, le : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff8000101212e4: ldr w1, [x19, #216] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff8000101212e8: ldr x2, [x19] : update_tg_cfs_runnable(): : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff8000101212ec: ldr x5, [x19, #224] : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff8000101212f0: lsr w1, w1, #10 : runnable_sum = max(runnable_sum, running_sum); 0.00 : ffff8000101212f4: cmp x1, x3 : se_weight(): 0.00 : ffff8000101212f8: lsr x2, x2, #10 : update_tg_cfs_runnable(): 0.00 : ffff8000101212fc: csel x3, x1, x3, ge // ge = tcont : se->avg.load_sum = runnable_sum; 0.00 : ffff800010121300: str x3, [x19, #200] : load_sum = (s64)se_weight(se) * runnable_sum; 0.00 : ffff800010121304: mul x1, x2, x3 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 0.00 : ffff800010121308: msub x2, x2, x0, x1 : div_s64_rem(): 0.00 : ffff80001012130c: lsr x0, x1, #1 0.00 : ffff800010121310: mov x1, #0x5041 // #20545 0.00 : ffff800010121314: movk x1, #0xafb, lsl #16 0.00 : ffff800010121318: movk x1, #0x400, lsl #32 0.00 : ffff80001012131c: movk x1, #0xafb5, lsl #48 0.00 : ffff800010121320: umulh x0, x0, x1 0.00 : ffff800010121324: lsr x0, x0, #14 : update_tg_cfs_runnable(): : se->avg.load_avg = load_avg; 0.00 : ffff800010121328: str x0, [x19, #224] : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff80001012132c: sub x0, x0, x5 : div_s64_rem(): 0.00 : ffff800010121330: mov x5, #0x5041 // #20545 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121334: ldr x1, [x4, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_avg, delta_avg); 0.00 : ffff800010121338: cmp x0, #0x0 : div_s64_rem(): 0.00 : ffff80001012133c: movk x5, #0xafb, lsl #16 : update_tg_cfs_runnable(): 0.00 : ffff800010121340: add x0, x0, x1 : div_s64_rem(): 0.00 : ffff800010121344: movk x5, #0x400, lsl #32 : update_tg_cfs_runnable(): 0.00 : ffff800010121348: ccmp x1, x0, #0x2, lt // lt = tstop : div_s64_rem(): 0.00 : ffff80001012134c: movk x5, #0xafb5, lsl #48 : update_tg_cfs_runnable(): 0.00 : ffff800010121350: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010121354: str x0, [x4, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_sum, delta_sum); 0.00 : ffff800010121358: cmp x2, #0x0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012135c: ldr x1, [x4, #136] : update_tg_cfs_runnable(): 0.00 : ffff800010121360: add x0, x2, x1 0.00 : ffff800010121364: ccmp x1, x0, #0x2, lt // lt = tstop 0.00 : ffff800010121368: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012136c: str x0, [x4, #136] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff800010121370: ldr x1, [x19, #16] : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff800010121374: ldr w2, [x19, #64] : se_runnable(): 0.00 : ffff800010121378: lsr x1, x1, #10 : update_tg_cfs_runnable(): : runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 0.00 : ffff80001012137c: mul x1, x1, x3 : div_s64_rem(): 0.00 : ffff800010121380: lsr x0, x1, #1 0.00 : ffff800010121384: umulh x0, x0, x5 0.00 : ffff800010121388: lsr x0, x0, #14 : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff80001012138c: cbz w2, ffff8000101213d8 : delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 0.00 : ffff800010121390: ldr x6, [x19, #232] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121394: ldr x7, [x4, #168] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010121398: ldr x2, [x19] : update_tg_cfs_runnable(): 0.00 : ffff80001012139c: sub x6, x0, x6 : se_weight(se) * se->avg.runnable_load_sum; 0.00 : ffff8000101213a0: ldr x8, [x19, #208] : add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 0.00 : ffff8000101213a4: add x5, x6, x7 0.00 : ffff8000101213a8: cmp x6, #0x0 0.00 : ffff8000101213ac: ccmp x7, x5, #0x2, lt // lt = tstop : se_weight(): 0.00 : ffff8000101213b0: lsr x2, x2, #10 : update_tg_cfs_runnable(): 0.00 : ffff8000101213b4: csel x5, x5, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101213b8: str x5, [x4, #168] : update_tg_cfs_runnable(): : delta_sum = runnable_load_sum - 0.00 : ffff8000101213bc: msub x1, x2, x8, x1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101213c0: ldr x2, [x4, #144] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 0.00 : ffff8000101213c4: cmp x1, #0x0 0.00 : ffff8000101213c8: add x1, x1, x2 0.00 : ffff8000101213cc: ccmp x2, x1, #0x2, lt // lt = tstop 0.00 : ffff8000101213d0: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101213d4: str x1, [x4, #144] : update_tg_cfs_runnable(): : se->avg.runnable_load_sum = runnable_sum; 0.00 : ffff8000101213d8: str x3, [x19, #208] : se->avg.runnable_load_avg = runnable_load_avg; 0.00 : ffff8000101213dc: str x0, [x19, #232] : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff8000101213e0: ldr x1, [x20, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff8000101213e4: add x0, x1, #0x80 0.00 : ffff8000101213e8: cmp x20, x0 0.00 : ffff8000101213ec: b.eq ffff800010121440 // b.none : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff8000101213f0: ldr x0, [x20, #336] 0.00 : ffff8000101213f4: ldr x1, [x29, #120] 0.00 : ffff8000101213f8: cmp x0, x1 0.00 : ffff8000101213fc: b.eq ffff8000101210b0 // b.none : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 0.00 : ffff800010121400: ldr x1, [x20, #160] 0.00 : ffff800010121404: ldr x2, [x20, #256] : if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 0.00 : ffff800010121408: subs x1, x1, x2 0.00 : ffff80001012140c: cneg x3, x1, mi // mi = first 0.00 : ffff800010121410: cmp x3, x2, lsr #6 0.00 : ffff800010121414: b.ls ffff8000101210b0 // b.plast : atomic_long_add(delta, &cfs_rq->tg->load_avg); 0.00 : ffff800010121418: add x2, x0, #0x140 : arch_static_branch_jump(): 0.00 : ffff80001012141c: b ffff8000101210a0 0.00 : ffff800010121420: b ffff8000101210a0 : __lse_atomic64_add(): 0.00 : ffff800010121424: stadd x1, [x2] 0.00 : ffff800010121428: b ffff8000101210a8 : update_load_avg(): : } else if (decayed) { 0.00 : ffff80001012142c: cbz w0, ffff8000101210b0 : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010121430: ldr x1, [x20, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff800010121434: add x0, x1, #0x80 0.00 : ffff800010121438: cmp x20, x0 0.00 : ffff80001012143c: b.ne ffff8000101213f0 // b.any : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff800010121440: ldrsw x3, [x1, #2568] 0.00 : ffff800010121444: adrp x2, ffff800011909000 0.00 : ffff800010121448: add x2, x2, #0x928 0.00 : ffff80001012144c: ldr x0, [x29, #104] 0.00 : ffff800010121450: ldr x2, [x2, x3, lsl #3] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121454: ldr x0, [x0, x2] : cpufreq_update_util(): : if (data) 0.00 : ffff800010121458: cbz x0, ffff8000101213f0 : data->func(data, rq_clock(rq), flags); 0.00 : ffff80001012145c: ldr x3, [x0] 0.00 : ffff800010121460: mov w2, #0x0 // #0 0.00 : ffff800010121464: ldr x1, [x1, #2400] 0.00 : ffff800010121468: blr x3 0.00 : ffff80001012146c: b ffff8000101213f0 : update_tg_cfs_runnable(): : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010121470: ldr x0, [x1] : u64 runnable_load_sum, load_sum = 0; 0.00 : ffff800010121474: mov x3, #0x0 // #0 : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010121478: lsr x0, x0, #10 0.00 : ffff80001012147c: cbz x0, ffff80001012148c : div_s64_rem(): 0.00 : ffff800010121480: ldr x3, [x1, #136] : *remainder = dividend % divisor; 0.00 : ffff800010121484: sxtw x0, w0 : return dividend / divisor; 0.00 : ffff800010121488: sdiv x3, x3, x0 : update_tg_cfs_runnable(): : runnable_sum = min(se->avg.load_sum, load_sum); 0.00 : ffff80001012148c: ldr x0, [x19, #200] 0.00 : ffff800010121490: cmp x0, x3 0.00 : ffff800010121494: csel x3, x0, x3, ls // ls = plast 0.00 : ffff800010121498: b ffff800010120cec : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff80001012149c: ldr x0, [x1] : u64 runnable_load_sum, load_sum = 0; 0.00 : ffff8000101214a0: mov x3, #0x0 // #0 : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff8000101214a4: lsr x0, x0, #10 0.00 : ffff8000101214a8: cbz x0, ffff8000101214b8 : div_s64_rem(): 0.00 : ffff8000101214ac: ldr x3, [x1, #136] : *remainder = dividend % divisor; 0.00 : ffff8000101214b0: sxtw x0, w0 : return dividend / divisor; 0.00 : ffff8000101214b4: sdiv x3, x3, x0 : update_tg_cfs_runnable(): : runnable_sum = min(se->avg.load_sum, load_sum); 0.00 : ffff8000101214b8: ldr x0, [x19, #200] 0.00 : ffff8000101214bc: cmp x0, x3 0.00 : ffff8000101214c0: csel x3, x0, x3, ls // ls = plast 0.00 : ffff8000101214c4: b ffff8000101212e4 : add_nr_running(): : if (!READ_ONCE(rq->rd->overload)) 0.00 : ffff8000101214c8: ldr x2, [x24, #2464] : __read_once_size(): 0.00 : ffff8000101214cc: ldr w1, [x2, #88] : add_nr_running(): 0.00 : ffff8000101214d0: cbnz w1, ffff8000101210dc : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101214d4: str w0, [x2, #88] 0.00 : ffff8000101214d8: b ffff8000101210dc : list_add_tail_rcu(): : __list_add_rcu(new, head->prev, head); 0.00 : ffff8000101214dc: add x3, x1, #0xa00 0.00 : ffff8000101214e0: ldur x4, [x3, #-232] : list_add_leaf_cfs_rq(): : list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 0.00 : ffff8000101214e4: add x0, x1, #0x910 : __list_add_rcu(): : new->prev = prev; 0.00 : ffff8000101214e8: stp x0, x4, [x20, #320] : rcu_assign_pointer(list_next_rcu(prev), new); 0.00 : ffff8000101214ec: stlr x2, [x4] : next->prev = new; 0.00 : ffff8000101214f0: stur x2, [x3, #-232] : list_add_leaf_cfs_rq(): : rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 0.00 : ffff8000101214f4: str x0, [x1, #2336] 0.00 : ffff8000101214f8: b ffff800010120ed4 : add_nr_running(): 0.00 : ffff8000101214fc: ldr x2, [x24, #2464] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121500: ldr w1, [x2, #88] : add_nr_running(): 0.00 : ffff800010121504: cbnz w1, ffff800010121510 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010121508: str w0, [x2, #88] 0.00 : ffff80001012150c: b ffff800010120ff4 0.00 : ffff800010121510: ldp x25, x26, [x29, #64] 0.00 : ffff800010121514: ldp x27, x28, [x29, #80] 0.00 : ffff800010121518: b ffff800010121000 0.00 : ffff80001012151c: ldr x2, [x24, #2464] 0.00 : ffff800010121520: b ffff800010121000 Percent | Source code & Disassembly of vmlinux for cycles (64800 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001011b180 : : irqtime_account_irq(): : /* : * Called before incrementing preempt_count on {soft,}irq_enter : * and before decrementing preempt_count on {soft,}irq_exit. : */ : void irqtime_account_irq(struct task_struct *curr) : { 0.40 : ffff80001011b180: stp x29, x30, [sp, #-64]! : struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); : s64 delta; : int cpu; : : if (!sched_clock_irqtime) 0.00 : ffff80001011b184: adrp x1, ffff800011afb000 : { 0.00 : ffff80001011b188: mov x29, sp : if (!sched_clock_irqtime) 0.51 : ffff80001011b18c: ldr w1, [x1, #3992] : { 92.61 : ffff80001011b190: stp x19, x20, [sp, #16] : struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); 0.00 : ffff80001011b194: adrp x19, ffff80001151f000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.05 : ffff80001011b198: mrs x20, tpidr_el1 : irqtime_account_irq(): 0.02 : ffff80001011b19c: add x19, x19, #0x40 : if (!sched_clock_irqtime) 0.00 : ffff80001011b1a0: cbnz w1, ffff80001011b1b0 : */ : if (hardirq_count()) : irqtime_account_delta(irqtime, delta, CPUTIME_IRQ); : else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) : irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ); : } 0.00 : ffff80001011b1a4: ldp x19, x20, [sp, #16] 0.00 : ffff80001011b1a8: ldp x29, x30, [sp], #64 0.00 : ffff80001011b1ac: ret 0.39 : ffff80001011b1b0: stp x21, x22, [x29, #32] : cpu = smp_processor_id(); 0.00 : ffff80001011b1b4: adrp x1, ffff80001151d000 0.31 : ffff80001011b1b8: str x23, [x29, #48] 0.00 : ffff80001011b1bc: mov x22, x0 0.00 : ffff80001011b1c0: add x1, x1, #0x18 : struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); 0.00 : ffff80001011b1c4: add x23, x19, x20 : delta = sched_clock_cpu(cpu) - irqtime->irq_start_time; 0.01 : ffff80001011b1c8: ldr w0, [x1, x20] 0.00 : ffff80001011b1cc: bl ffff80001011b140 0.00 : ffff80001011b1d0: ldr x21, [x23, #16] : irqtime->irq_start_time += delta; 1.09 : ffff80001011b1d4: str x0, [x23, #16] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.27 : ffff80001011b1d8: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011b1dc: ldr w2, [x1, #16] : irqtime_account_irq(): : delta = sched_clock_cpu(cpu) - irqtime->irq_start_time; 0.00 : ffff80001011b1e0: sub x21, x0, x21 : if (hardirq_count()) 0.00 : ffff80001011b1e4: tst w2, #0xf0000 0.00 : ffff80001011b1e8: b.ne ffff80001011b208 // b.any : __read_once_size(): 0.01 : ffff80001011b1ec: ldr w0, [x1, #16] : irqtime_account_irq(): : else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) 0.00 : ffff80001011b1f0: tbnz w0, #8, ffff80001011b250 0.29 : ffff80001011b1f4: ldp x21, x22, [x29, #32] 0.06 : ffff80001011b1f8: ldr x23, [x29, #48] : } 0.00 : ffff80001011b1fc: ldp x19, x20, [sp, #16] 0.00 : ffff80001011b200: ldp x29, x30, [sp], #64 0.00 : ffff80001011b204: ret : __my_cpu_offset(): 0.00 : ffff80001011b208: mrs x0, tpidr_el1 : irqtime_account_delta(): : u64 *cpustat = kcpustat_this_cpu->cpustat; 0.39 : ffff80001011b20c: adrp x1, ffff80001151e000 0.00 : ffff80001011b210: add x1, x1, #0xfc0 0.00 : ffff80001011b214: add x1, x1, x0 : cpustat[idx] += delta; 0.00 : ffff80001011b218: ldr x0, [x1, #32] 0.00 : ffff80001011b21c: add x0, x0, x21 1.98 : ffff80001011b220: str x0, [x1, #32] : irqtime->total += delta; 0.77 : ffff80001011b224: ldr x0, [x19, x20] 0.00 : ffff80001011b228: add x0, x0, x21 0.01 : ffff80001011b22c: str x0, [x19, x20] : irqtime->tick_delta += delta; 0.00 : ffff80001011b230: ldr x0, [x23, #8] 0.00 : ffff80001011b234: add x0, x0, x21 0.31 : ffff80001011b238: str x0, [x23, #8] 0.14 : ffff80001011b23c: ldp x21, x22, [x29, #32] 0.02 : ffff80001011b240: ldr x23, [x29, #48] : irqtime_account_irq(): : } 0.00 : ffff80001011b244: ldp x19, x20, [sp, #16] 0.33 : ffff80001011b248: ldp x29, x30, [sp], #64 0.00 : ffff80001011b24c: ret 0.00 : ffff80001011b250: str x24, [x29, #56] : __read_once_size(): 0.00 : ffff80001011b254: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001011b258: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001011b25c: str w0, [x1, #16] : this_cpu_ksoftirqd(): : : DECLARE_PER_CPU(struct task_struct *, ksoftirqd); : : static inline struct task_struct *this_cpu_ksoftirqd(void) : { : return this_cpu_read(ksoftirqd); 0.00 : ffff80001011b260: adrp x0, ffff80001151e000 0.00 : ffff80001011b264: add x0, x0, #0xf80 : __my_cpu_offset(): 0.00 : ffff80001011b268: mrs x2, tpidr_el1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011b26c: ldr x24, [x0, x2] 0.00 : ffff80001011b270: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001011b274: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001011b278: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001011b27c: cbnz x0, ffff80001011b2cc : this_cpu_ksoftirqd(): 0.00 : ffff80001011b280: bl ffff800010cd2e78 : irqtime_account_irq(): : else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) 0.00 : ffff80001011b284: cmp x22, x24 0.00 : ffff80001011b288: b.eq ffff80001011b2d8 // b.none : __my_cpu_offset(): 0.00 : ffff80001011b28c: mrs x1, tpidr_el1 : irqtime_account_delta(): : u64 *cpustat = kcpustat_this_cpu->cpustat; 0.00 : ffff80001011b290: adrp x0, ffff80001151e000 0.00 : ffff80001011b294: add x0, x0, #0xfc0 0.00 : ffff80001011b298: add x0, x0, x1 : cpustat[idx] += delta; 0.00 : ffff80001011b29c: ldr x1, [x0, #24] 0.00 : ffff80001011b2a0: add x1, x1, x21 0.00 : ffff80001011b2a4: str x1, [x0, #24] : irqtime->total += delta; 0.00 : ffff80001011b2a8: ldr x0, [x19, x20] 0.00 : ffff80001011b2ac: add x0, x0, x21 0.00 : ffff80001011b2b0: str x0, [x19, x20] : irqtime->tick_delta += delta; 0.00 : ffff80001011b2b4: ldr x0, [x23, #8] 0.00 : ffff80001011b2b8: add x0, x0, x21 0.00 : ffff80001011b2bc: str x0, [x23, #8] 0.00 : ffff80001011b2c0: ldp x21, x22, [x29, #32] 0.00 : ffff80001011b2c4: ldp x23, x24, [x29, #48] 0.00 : ffff80001011b2c8: b ffff80001011b1a4 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011b2cc: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001011b2d0: cbz x0, ffff80001011b280 0.02 : ffff80001011b2d4: b ffff80001011b284 0.00 : ffff80001011b2d8: ldp x21, x22, [x29, #32] 0.00 : ffff80001011b2dc: ldp x23, x24, [x29, #48] 0.00 : ffff80001011b2e0: b ffff80001011b1a4 Percent | Source code & Disassembly of vmlinux for cycles (55436 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff8000104b5338 : : gic_raise_softirq(): : pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); : gic_write_sgi1r(val); : } : : static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) : { 0.10 : ffff8000104b5338: stp x29, x30, [sp, #-96]! : int cpu; : : if (WARN_ON(irq >= 16)) 0.00 : ffff8000104b533c: cmp w1, #0xf : { 0.09 : ffff8000104b5340: mov x29, sp : if (WARN_ON(irq >= 16)) 0.00 : ffff8000104b5344: b.hi ffff8000104b5478 // b.pmore 0.24 : ffff8000104b5348: stp x19, x20, [x29, #16] 0.02 : ffff8000104b534c: stp x21, x22, [x29, #32] 0.00 : ffff8000104b5350: mov x21, x0 0.15 : ffff8000104b5354: stp x23, x24, [x29, #48] 0.09 : ffff8000104b5358: stp x25, x26, [x29, #64] 5.84 : ffff8000104b535c: str x27, [x29, #80] : : /* : * Ensure that stores to Normal memory are visible to the : * other CPUs before issuing the IPI. : */ : wmb(); 0.22 : ffff8000104b5360: dsb st 63.95 : ffff8000104b5364: adrp x24, ffff80001190a000 <__per_cpu_offset+0x6d8> : : for_each_cpu(cpu, mask) { 0.00 : ffff8000104b5368: mov w26, #0xffffffff // #-1 : gic_send_sgi(): : irq << ICC_SGI1R_SGI_ID_SHIFT | 0.00 : ffff8000104b536c: lsl w25, w1, #24 : gic_raise_softirq(): : for_each_cpu(cpu, mask) { 0.00 : ffff8000104b5370: mov w0, w26 0.00 : ffff8000104b5374: mov x1, x21 0.00 : ffff8000104b5378: bl ffff800010cb92d8 0.50 : ffff8000104b537c: ldr w3, [x24, #752] : u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 0.00 : ffff8000104b5380: adrp x22, ffff800011914000 : gic_compute_target_list(): : tlist |= 1 << (mpidr & 0xf); 0.00 : ffff8000104b5384: mov w23, #0x1 // #1 : gic_raise_softirq(): : u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 0.00 : ffff8000104b5388: add x22, x22, #0x980 : for_each_cpu(cpu, mask) { 0.34 : ffff8000104b538c: mov w26, w0 0.00 : ffff8000104b5390: add x20, x24, #0x2f0 0.00 : ffff8000104b5394: cmp w0, w3 0.00 : ffff8000104b5398: b.cs ffff8000104b5450 // b.hs, b.nlast 0.00 : ffff8000104b539c: nop : u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 1.98 : ffff8000104b53a0: ldr x1, [x22, w26, sxtw #3] : gic_compute_target_list(): : u16 tlist = 0; 0.00 : ffff8000104b53a4: mov w27, #0x0 // #0 : gic_raise_softirq(): : u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 0.00 : ffff8000104b53a8: and x19, x1, #0xfffffffffffffff0 0.00 : ffff8000104b53ac: b ffff8000104b53d8 : gic_compute_target_list(): : tlist |= 1 << (mpidr & 0xf); 0.00 : ffff8000104b53b0: and w27, w2, #0xffff : next_cpu = cpumask_next(cpu, mask); 0.00 : ffff8000104b53b4: bl ffff800010cb92d8 : if (next_cpu >= nr_cpu_ids) 0.26 : ffff8000104b53b8: ldr w3, [x20] 0.01 : ffff8000104b53bc: cmp w0, w3 0.00 : ffff8000104b53c0: b.cs ffff8000104b53f4 // b.hs, b.nlast : mpidr = cpu_logical_map(cpu); 0.00 : ffff8000104b53c4: ldr x1, [x22, w0, sxtw #3] : next_cpu = cpumask_next(cpu, mask); 0.00 : ffff8000104b53c8: mov w26, w0 : if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 0.00 : ffff8000104b53cc: and x0, x1, #0xfffffffffffffff0 0.00 : ffff8000104b53d0: cmp x19, x0 0.00 : ffff8000104b53d4: b.ne ffff8000104b5470 // b.any : tlist |= 1 << (mpidr & 0xf); 9.58 : ffff8000104b53d8: and w2, w1, #0xf : next_cpu = cpumask_next(cpu, mask); 0.00 : ffff8000104b53dc: mov w0, w26 0.00 : ffff8000104b53e0: mov x1, x21 : while (cpu < nr_cpu_ids) { 0.00 : ffff8000104b53e4: cmp w3, w26 : tlist |= 1 << (mpidr & 0xf); 0.00 : ffff8000104b53e8: lsl w2, w23, w2 0.00 : ffff8000104b53ec: orr w2, w2, w27 : while (cpu < nr_cpu_ids) { 0.00 : ffff8000104b53f0: b.hi ffff8000104b53b0 // b.pmore : gic_send_sgi(): : MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 0.02 : ffff8000104b53f4: lsr x0, x19, #8 : val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 0.00 : ffff8000104b53f8: lsr x1, x19, #32 : MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 0.01 : ffff8000104b53fc: lsr x2, x19, #16 : MPIDR_TO_SGI_RS(cluster_id) | 0.13 : ffff8000104b5400: lsl x19, x19, #40 : MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 0.01 : ffff8000104b5404: ubfiz x0, x0, #16, #8 : val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 0.04 : ffff8000104b5408: ubfiz x1, x1, #48, #8 : MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 0.06 : ffff8000104b540c: ubfiz x2, x2, #32, #8 : MPIDR_TO_SGI_RS(cluster_id) | 0.01 : ffff8000104b5410: and x19, x19, #0xf00000000000 : val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 0.05 : ffff8000104b5414: orr x19, x0, x19 : tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 0.01 : ffff8000104b5418: and x27, x27, #0xffff : val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 0.08 : ffff8000104b541c: orr x0, x1, x2 0.02 : ffff8000104b5420: orr x27, x27, x25 0.01 : ffff8000104b5424: orr x19, x19, x0 0.00 : ffff8000104b5428: orr x27, x19, x27 : gic_write_sgi1r(): 0.01 : ffff8000104b542c: msr s3_0_c12_c11_5, x27 : gic_raise_softirq(): : for_each_cpu(cpu, mask) { 0.18 : ffff8000104b5430: mov w0, w26 0.02 : ffff8000104b5434: mov x1, x21 0.00 : ffff8000104b5438: bl ffff800010cb92d8 0.13 : ffff8000104b543c: add x20, x24, #0x2f0 0.35 : ffff8000104b5440: ldr w3, [x24, #752] 0.00 : ffff8000104b5444: mov w26, w0 0.00 : ffff8000104b5448: cmp w0, w3 0.00 : ffff8000104b544c: b.cc ffff8000104b53a0 // b.lo, b.ul, b.last : tlist = gic_compute_target_list(&cpu, mask, cluster_id); : gic_send_sgi(cluster_id, tlist, irq); : } : : /* Force the above writes to ICC_SGI1R_EL1 to be executed */ : isb(); 0.17 : ffff8000104b5450: isb 9.01 : ffff8000104b5454: ldp x19, x20, [x29, #16] 5.44 : ffff8000104b5458: ldp x21, x22, [x29, #32] 0.00 : ffff8000104b545c: ldp x23, x24, [x29, #48] 0.46 : ffff8000104b5460: ldp x25, x26, [x29, #64] 0.00 : ffff8000104b5464: ldr x27, [x29, #80] : } 0.44 : ffff8000104b5468: ldp x29, x30, [sp], #96 0.00 : ffff8000104b546c: ret : gic_compute_target_list(): : cpu--; 0.00 : ffff8000104b5470: sub w26, w26, #0x1 0.00 : ffff8000104b5474: b ffff8000104b53f4 : gic_raise_softirq(): : if (WARN_ON(irq >= 16)) 0.00 : ffff8000104b5478: brk #0x800 : } 0.00 : ffff8000104b547c: ldp x29, x30, [sp], #96 0.00 : ffff8000104b5480: ret Percent | Source code & Disassembly of vmlinux for cycles (53821 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010252740 <__slab_free>: : __slab_free(): : */ : static void __slab_free(struct kmem_cache *s, struct page *page, : void *head, void *tail, int cnt, : unsigned long addr) : : { 0.02 : ffff800010252740: stp x29, x30, [sp, #-208]! 0.00 : ffff800010252744: adrp x6, ffff800011909000 0.00 : ffff800010252748: add x8, x6, #0x908 : kmem_cache_debug(): : return unlikely(s->flags & SLAB_DEBUG_FLAGS); 0.08 : ffff80001025274c: mov w6, #0xd00 // #3328 : __slab_free(): : { 0.14 : ffff800010252750: mov x29, sp 0.74 : ffff800010252754: str x19, [sp, #16] : kmem_cache_debug(): : return unlikely(s->flags & SLAB_DEBUG_FLAGS); 0.00 : ffff800010252758: movk w6, #0x21, lsl #16 : __slab_free(): : { 0.41 : ffff80001025275c: stp x21, x22, [sp, #32] 0.00 : ffff800010252760: mov x19, x1 0.05 : ffff800010252764: stp x23, x24, [sp, #48] 0.00 : ffff800010252768: mov x21, x0 0.27 : ffff80001025276c: str x2, [x29, #120] 0.00 : ffff800010252770: mov x23, x3 0.78 : ffff800010252774: ldr x7, [x8] 0.39 : ffff800010252778: str x7, [x29, #200] 0.01 : ffff80001025277c: mov x7, #0x0 // #0 : kmem_cache_debug(): : return unlikely(s->flags & SLAB_DEBUG_FLAGS); 0.07 : ffff800010252780: ldr w8, [x0, #8] : __slab_free(): : { 0.00 : ffff800010252784: mov w22, w4 : struct kmem_cache_node *n = NULL; : unsigned long uninitialized_var(flags); : : stat(s, FREE_SLOWPATH); : : if (kmem_cache_debug(s) && 0.00 : ffff800010252788: tst w8, w6 0.01 : ffff80001025278c: b.ne ffff800010252a30 <__slab_free+0x2f0> // b.any 0.30 : ffff800010252790: and w22, w22, #0xffff 0.95 : ffff800010252794: str x20, [x29, #24] 0.13 : ffff800010252798: stp x25, x26, [x29, #64] 0.12 : ffff80001025279c: stp x27, x28, [x29, #80] : if (unlikely(n)) { : spin_unlock_irqrestore(&n->list_lock, flags); : n = NULL; : } : prior = page->freelist; : counters = page->counters; 0.46 : ffff8000102527a0: ldp x25, x20, [x19, #32] : set_freepointer(s, tail, prior); : new.counters = counters; 1.03 : ffff8000102527a4: str x20, [x29, #168] : set_freepointer(): : unsigned long freeptr_addr = (unsigned long)object + s->offset; 0.11 : ffff8000102527a8: ldr w0, [x21, #32] : __slab_free(): 0.00 : ffff8000102527ac: mov x26, #0x0 // #0 : was_frozen = new.frozen; : new.inuse -= cnt; 0.00 : ffff8000102527b0: sub w6, w20, w22 : if ((!new.inuse || !prior) && !was_frozen) { 0.00 : ffff8000102527b4: cmp x25, #0x0 : new.inuse -= cnt; 0.27 : ffff8000102527b8: and w27, w6, #0xffff : was_frozen = new.frozen; 0.00 : ffff8000102527bc: lsr w1, w20, #24 : new.inuse -= cnt; 0.17 : ffff8000102527c0: strh w27, [x29, #168] : if ((!new.inuse || !prior) && !was_frozen) { 0.00 : ffff8000102527c4: cset w24, eq // eq = none : set_freepointer(): : *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr); 0.74 : ffff8000102527c8: str x25, [x23, x0] : __slab_free(): : if ((!new.inuse || !prior) && !was_frozen) { 0.00 : ffff8000102527cc: cmp w27, #0x0 : was_frozen = new.frozen; 0.00 : ffff8000102527d0: ubfx x28, x1, #7, #1 : if ((!new.inuse || !prior) && !was_frozen) { 0.00 : ffff8000102527d4: ccmp w24, #0x0, #0x0, ne // ne = any 1.08 : ffff8000102527d8: ldr w0, [x21, #8] 0.00 : ffff8000102527dc: b.eq ffff800010252818 <__slab_free+0xd8> // b.none 0.08 : ffff8000102527e0: cbnz w28, ffff800010252818 <__slab_free+0xd8> : : if (kmem_cache_has_cpu_partial(s) && !prior) { 0.00 : ffff8000102527e4: mov w2, #0xd00 // #3328 0.00 : ffff8000102527e8: movk w2, #0x21, lsl #16 0.00 : ffff8000102527ec: tst w0, w2 0.00 : ffff8000102527f0: ccmp w24, #0x0, #0x4, eq // eq = none 0.00 : ffff8000102527f4: b.ne ffff8000102529f8 <__slab_free+0x2b8> // b.any : page_to_nid(): : #else : static inline int page_to_nid(const struct page *page) : { : struct page *p = (struct page *)page; : : return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK; 0.00 : ffff8000102527f8: ldr x0, [x19] : get_node(): : : }; : : static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) : { : return s->node[node]; 0.00 : ffff8000102527fc: lsr x0, x0, #62 0.00 : ffff800010252800: add x0, x0, #0x30 0.00 : ffff800010252804: ldr x26, [x21, x0, lsl #3] : __slab_free(): : * drop the list_lock without any processing. : * : * Otherwise the list_lock will synchronize with : * other processors updating the list of slabs. : */ : spin_lock_irqsave(&n->list_lock, flags); 0.00 : ffff800010252808: mov x0, x26 0.00 : ffff80001025280c: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff800010252810: str x0, [x29, #104] 0.00 : ffff800010252814: ldr w0, [x21, #8] : : } : } : : } while (!cmpxchg_double_slab(s, page, 2.36 : ffff800010252818: ldr x3, [x29, #168] : cmpxchg_double_slab(): : if (s->flags & __CMPXCHG_DOUBLE) { 0.00 : ffff80001025281c: tbz w0, #30, ffff800010252868 <__slab_free+0x128> : if (cmpxchg_double(&page->freelist, &page->counters, 20.05 : ffff800010252820: add x4, x19, #0x20 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010252824: b ffff8000102528e4 <__slab_free+0x1a4> 0.00 : ffff800010252828: b ffff8000102528e4 <__slab_free+0x1a4> : __lse__cmpxchg_double_mb(): : \ : return x0; \ : } : : __CMPXCHG_DBL( , ) : __CMPXCHG_DBL(_mb, al, "memory") 0.00 : ffff80001025282c: mov x0, x25 0.00 : ffff800010252830: mov x1, x20 1.89 : ffff800010252834: ldr x2, [x29, #120] 3.31 : ffff800010252838: caspal x0, x1, x2, x3, [x4] 53.28 : ffff80001025283c: eor x0, x0, x25 0.00 : ffff800010252840: eor x1, x1, x20 0.00 : ffff800010252844: orr x0, x0, x1 : cmpxchg_double_slab(): 0.00 : ffff800010252848: cbz x0, ffff8000102528f4 <__slab_free+0x1b4> 0.00 : ffff80001025284c: nop : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff800010252850: yield : __slab_free(): : if (unlikely(n)) { 0.00 : ffff800010252854: cbz x26, ffff8000102527a0 <__slab_free+0x60> : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff800010252858: ldr x1, [x29, #104] 0.00 : ffff80001025285c: mov x0, x26 0.00 : ffff800010252860: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010252864: b ffff8000102527a0 <__slab_free+0x60> : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010252868: mrs x0, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001025286c: and w1, w0, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff800010252870: cbnz w1, ffff80001025287c <__slab_free+0x13c> : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010252874: mov x1, #0x60 // #96 0.00 : ffff800010252878: msr daifset, #0x2 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001025287c: mrs x2, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010252880: ldr w1, [x2, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010252884: add w1, w1, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010252888: str w1, [x2, #16] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001025288c: ldr x1, [x19] : test_and_set_bit_lock(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (READ_ONCE(*p) & mask) 0.00 : ffff800010252890: tbz w1, #0, ffff800010252940 <__slab_free+0x200> : get_current(): 0.00 : ffff800010252894: mrs x2, sp_el0 : __read_once_size(): 0.00 : ffff800010252898: ldr x1, [x2, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001025289c: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102528a0: str w1, [x2, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102528a4: cbz x1, ffff8000102528b0 <__slab_free+0x170> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102528a8: ldr x1, [x2, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102528ac: cbnz x1, ffff8000102528c8 <__slab_free+0x188> 0.00 : ffff8000102528b0: str x0, [x29, #96] 0.00 : ffff8000102528b4: str x3, [x29, #112] : bit_spin_lock(): : * attempt to acquire the lock bit. : */ : preempt_disable(); : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) : while (unlikely(test_and_set_bit_lock(bitnum, addr))) { : preempt_enable(); 0.00 : ffff8000102528b8: bl ffff800010cd2e50 0.00 : ffff8000102528bc: ldr x0, [x29, #96] 0.00 : ffff8000102528c0: ldr x3, [x29, #112] 0.00 : ffff8000102528c4: nop : cpu_relax(): 0.00 : ffff8000102528c8: yield : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000102528cc: ldr x1, [x19] : bit_spin_lock(): : do { : cpu_relax(); : } while (test_bit(bitnum, addr)); 0.00 : ffff8000102528d0: tbz w1, #0, ffff80001025287c <__slab_free+0x13c> : cpu_relax(): 0.00 : ffff8000102528d4: yield : test_bit(): 0.00 : ffff8000102528d8: ldr x1, [x19] : bit_spin_lock(): 0.00 : ffff8000102528dc: tbnz w1, #0, ffff8000102528c8 <__slab_free+0x188> 0.00 : ffff8000102528e0: b ffff80001025287c <__slab_free+0x13c> : __ll_sc__cmpxchg_double_mb(): : \ : return ret; \ : } : : __CMPXCHG_DBL( , , , ) : __CMPXCHG_DBL(_mb, dmb ish, l, "memory") 0.00 : ffff8000102528e4: add x2, x19, #0x20 0.00 : ffff8000102528e8: ldr x4, [x29, #120] 0.00 : ffff8000102528ec: b ffff800010255b3c : cmpxchg_double_slab(): : if (cmpxchg_double(&page->freelist, &page->counters, 0.00 : ffff8000102528f0: cbnz x0, ffff800010252850 <__slab_free+0x110> : __slab_free(): : prior, counters, : head, new.counters, : "__slab_free")); : : if (likely(!n)) { 0.01 : ffff8000102528f4: cbnz x26, ffff800010252a3c <__slab_free+0x2fc> : : /* : * If we just froze the page then put it onto the : * per cpu partial list. : */ : if (new.frozen && !was_frozen) { 5.45 : ffff8000102528f8: ldrb w0, [x29, #171] 0.00 : ffff8000102528fc: eor w10, w28, #0x1 0.00 : ffff800010252900: tst w10, w0, lsr #7 0.00 : ffff800010252904: b.ne ffff800010252a10 <__slab_free+0x2d0> // b.any 1.61 : ffff800010252908: ldr x20, [x29, #24] 0.49 : ffff80001025290c: ldp x25, x26, [x29, #64] 1.17 : ffff800010252910: ldp x27, x28, [x29, #80] : } : : spin_unlock_irqrestore(&n->list_lock, flags); : stat(s, FREE_SLAB); : discard_slab(s, page); : } 0.00 : ffff800010252914: adrp x0, ffff800011909000 0.00 : ffff800010252918: add x24, x0, #0x908 0.09 : ffff80001025291c: ldr x1, [x29, #200] 0.11 : ffff800010252920: ldr x0, [x24] 0.00 : ffff800010252924: eor x0, x1, x0 0.00 : ffff800010252928: cbnz x0, ffff800010252adc <__slab_free+0x39c> 0.36 : ffff80001025292c: ldr x19, [sp, #16] 1.33 : ffff800010252930: ldp x21, x22, [sp, #32] 0.04 : ffff800010252934: ldp x23, x24, [sp, #48] 0.03 : ffff800010252938: ldp x29, x30, [sp], #208 0.00 : ffff80001025293c: ret : arch_static_branch_jump(): 0.00 : ffff800010252940: b ffff8000102529f0 <__slab_free+0x2b0> 0.00 : ffff800010252944: b ffff8000102529f0 <__slab_free+0x2b0> : __lse_atomic64_fetch_or_acquire(): : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff800010252948: mov x1, #0x1 // #1 0.00 : ffff80001025294c: ldseta x1, x1, [x19] : bit_spin_lock(): : while (unlikely(test_and_set_bit_lock(bitnum, addr))) { 0.00 : ffff800010252950: tbnz w1, #0, ffff800010252894 <__slab_free+0x154> : cmpxchg_double_slab(): : if (page->freelist == freelist_old && 0.00 : ffff800010252954: ldr x1, [x19, #32] 0.00 : ffff800010252958: cmp x25, x1 0.00 : ffff80001025295c: b.eq ffff8000102529a0 <__slab_free+0x260> // b.none : __read_once_size(): 0.00 : ffff800010252960: ldr x1, [x19] : __clear_bit_unlock(): : { : unsigned long old; : : p += BIT_WORD(nr); : old = READ_ONCE(*p); : old &= ~BIT_MASK(nr); 0.00 : ffff800010252964: and x1, x1, #0xfffffffffffffffe : atomic64_set_release(): : : #ifndef atomic64_set_release : static inline void : atomic64_set_release(atomic64_t *v, s64 i) : { : smp_store_release(&(v)->counter, i); 0.00 : ffff800010252968: stlr x1, [x19] : get_current(): 0.00 : ffff80001025296c: mrs x2, sp_el0 : __read_once_size(): 0.00 : ffff800010252970: ldr x1, [x2, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010252974: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010252978: str w1, [x2, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001025297c: cbz x1, ffff800010252988 <__slab_free+0x248> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010252980: ldr x1, [x2, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010252984: cbnz x1, ffff800010252994 <__slab_free+0x254> 0.00 : ffff800010252988: str x0, [x29, #112] : __bit_spin_unlock(): : BUG_ON(!test_bit(bitnum, addr)); : #endif : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) : __clear_bit_unlock(bitnum, addr); : #endif : preempt_enable(); 0.00 : ffff80001025298c: bl ffff800010cd2e50 0.00 : ffff800010252990: ldr x0, [x29, #112] : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff800010252994: msr daif, x0 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010252998: nop 0.00 : ffff80001025299c: b ffff800010252850 <__slab_free+0x110> : cmpxchg_double_slab(): 0.00 : ffff8000102529a0: ldr x1, [x19, #40] 0.00 : ffff8000102529a4: cmp x20, x1 0.00 : ffff8000102529a8: b.ne ffff800010252960 <__slab_free+0x220> // b.any : __read_once_size(): 0.00 : ffff8000102529ac: ldr x1, [x19] : cmpxchg_double_slab(): : page->freelist = freelist_new; 0.00 : ffff8000102529b0: ldr x2, [x29, #120] : __clear_bit_unlock(): 0.00 : ffff8000102529b4: and x1, x1, #0xfffffffffffffffe : cmpxchg_double_slab(): : page->counters = counters_new; 0.00 : ffff8000102529b8: stp x2, x3, [x19, #32] : atomic64_set_release(): 0.00 : ffff8000102529bc: stlr x1, [x19] : get_current(): 0.00 : ffff8000102529c0: mrs x2, sp_el0 : __read_once_size(): 0.00 : ffff8000102529c4: ldr x1, [x2, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102529c8: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102529cc: str w1, [x2, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102529d0: cbnz x1, ffff800010252a04 <__slab_free+0x2c4> 0.00 : ffff8000102529d4: str x0, [x29, #120] : __bit_spin_unlock(): 0.00 : ffff8000102529d8: bl ffff800010cd2e50 0.00 : ffff8000102529dc: ldr x0, [x29, #120] : cmpxchg_double_slab(): : local_irq_restore(flags); 0.00 : ffff8000102529e0: bl ffff80001024c9f0 0.00 : ffff8000102529e4: b ffff8000102528f4 <__slab_free+0x1b4> : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff8000102529e8: dsb sy 0.00 : ffff8000102529ec: b ffff800010252850 <__slab_free+0x110> : __ll_sc_atomic64_fetch_or_acquire(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000102529f0: b ffff800010255b64 0.00 : ffff8000102529f4: b ffff800010252950 <__slab_free+0x210> : __slab_free(): : new.frozen = 1; 0.00 : ffff8000102529f8: orr w1, w1, #0xffffff80 0.00 : ffff8000102529fc: strb w1, [x29, #171] 0.00 : ffff800010252a00: b ffff800010252818 <__slab_free+0xd8> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010252a04: ldr x1, [x2, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010252a08: cbz x1, ffff8000102529d4 <__slab_free+0x294> 0.00 : ffff800010252a0c: b ffff8000102529e0 <__slab_free+0x2a0> : __slab_free(): : put_cpu_partial(s, page, 1); 0.00 : ffff800010252a10: mov w2, #0x1 // #1 0.00 : ffff800010252a14: mov x1, x19 0.00 : ffff800010252a18: mov x0, x21 0.00 : ffff800010252a1c: bl ffff800010251418 0.00 : ffff800010252a20: ldr x20, [x29, #24] 0.00 : ffff800010252a24: ldp x25, x26, [x29, #64] 0.00 : ffff800010252a28: ldp x27, x28, [x29, #80] 0.00 : ffff800010252a2c: b ffff800010252914 <__slab_free+0x1d4> : !free_debug_processing(s, page, head, tail, cnt, addr)) 0.00 : ffff800010252a30: bl ffff8000102507c0 : if (kmem_cache_debug(s) && 0.00 : ffff800010252a34: cbz w0, ffff800010252914 <__slab_free+0x1d4> 0.00 : ffff800010252a38: b ffff800010252790 <__slab_free+0x50> : if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) 0.00 : ffff800010252a3c: cbz w27, ffff800010252a74 <__slab_free+0x334> : kmem_cache_has_cpu_partial(): : return !kmem_cache_debug(s); 0.00 : ffff800010252a40: ldr w0, [x21, #8] : kmem_cache_debug(): : return unlikely(s->flags & SLAB_DEBUG_FLAGS); 0.00 : ffff800010252a44: mov w1, #0xd00 // #3328 0.00 : ffff800010252a48: movk w1, #0x21, lsl #16 : __slab_free(): : if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { 0.00 : ffff800010252a4c: tst w0, w1 0.00 : ffff800010252a50: ccmp w24, #0x0, #0x4, ne // ne = any 0.00 : ffff800010252a54: b.ne ffff800010252aec <__slab_free+0x3ac> // b.any : spin_unlock_irqrestore(): 0.00 : ffff800010252a58: ldr x1, [x29, #104] 0.00 : ffff800010252a5c: mov x0, x26 0.00 : ffff800010252a60: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010252a64: ldr x20, [x29, #24] 0.00 : ffff800010252a68: ldp x25, x26, [x29, #64] 0.00 : ffff800010252a6c: ldp x27, x28, [x29, #80] 0.00 : ffff800010252a70: b ffff800010252914 <__slab_free+0x1d4> : __slab_free(): : if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) 0.00 : ffff800010252a74: ldr x0, [x21, #16] 0.00 : ffff800010252a78: ldr x1, [x26, #8] 0.00 : ffff800010252a7c: cmp x1, x0 0.00 : ffff800010252a80: b.cc ffff800010252a40 <__slab_free+0x300> // b.lo, b.ul, b.last : if (prior) { 0.00 : ffff800010252a84: cbz x25, ffff800010252b18 <__slab_free+0x3d8> : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff800010252a88: ldp x1, x0, [x19, #8] : __list_del(): : next->prev = prev; 0.00 : ffff800010252a8c: str x0, [x1, #8] : list_del(): : * in an undefined state. : */ : static inline void list_del(struct list_head *entry) : { : __list_del_entry(entry); : entry->next = LIST_POISON1; 0.00 : ffff800010252a90: mov x3, #0x100 // #256 : entry->prev = LIST_POISON2; 0.00 : ffff800010252a94: mov x2, #0x122 // #290 : entry->next = LIST_POISON1; 0.00 : ffff800010252a98: movk x3, #0xdead, lsl #48 : entry->prev = LIST_POISON2; 0.00 : ffff800010252a9c: movk x2, #0xdead, lsl #48 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010252aa0: str x1, [x0] : list_del(): 0.00 : ffff800010252aa4: stp x3, x2, [x19, #8] : remove_partial(): : n->nr_partial--; 0.00 : ffff800010252aa8: ldr x0, [x26, #8] 0.00 : ffff800010252aac: sub x0, x0, #0x1 0.00 : ffff800010252ab0: str x0, [x26, #8] : spin_unlock_irqrestore(): 0.00 : ffff800010252ab4: ldr x1, [x29, #104] 0.00 : ffff800010252ab8: mov x0, x26 0.00 : ffff800010252abc: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : __slab_free(): : discard_slab(s, page); 0.00 : ffff800010252ac0: mov x1, x19 0.00 : ffff800010252ac4: mov x0, x21 0.00 : ffff800010252ac8: bl ffff80001024fca0 0.00 : ffff800010252acc: ldr x20, [x29, #24] 0.00 : ffff800010252ad0: ldp x25, x26, [x29, #64] 0.00 : ffff800010252ad4: ldp x27, x28, [x29, #80] 0.00 : ffff800010252ad8: b ffff800010252914 <__slab_free+0x1d4> 0.00 : ffff800010252adc: str x20, [x29, #24] 0.00 : ffff800010252ae0: stp x25, x26, [x29, #64] 0.00 : ffff800010252ae4: stp x27, x28, [x29, #80] : } 0.00 : ffff800010252ae8: bl ffff8000100e64d0 <__stack_chk_fail> : remove_full(): : if (!(s->flags & SLAB_STORE_USER)) 0.00 : ffff800010252aec: tbnz w0, #16, ffff800010252b44 <__slab_free+0x404> : __add_partial(): : n->nr_partial++; 0.00 : ffff800010252af0: ldr x0, [x26, #8] : list_add_tail(&page->slab_list, &n->partial); 0.00 : ffff800010252af4: add x2, x19, #0x8 : list_add_tail(): : __list_add(new, head->prev, head); 0.00 : ffff800010252af8: ldr x1, [x26, #24] : __add_partial(): : n->nr_partial++; 0.00 : ffff800010252afc: add x0, x0, #0x1 0.00 : ffff800010252b00: str x0, [x26, #8] : __list_add(): : next->prev = new; 0.00 : ffff800010252b04: str x2, [x26, #24] : __add_partial(): : list_add_tail(&page->slab_list, &n->partial); 0.00 : ffff800010252b08: add x0, x26, #0x10 : __list_add(): : new->prev = prev; 0.00 : ffff800010252b0c: stp x0, x1, [x19, #8] : __write_once_size(): 0.00 : ffff800010252b10: str x2, [x1] 0.00 : ffff800010252b14: b ffff800010252a58 <__slab_free+0x318> : remove_full(): : if (!(s->flags & SLAB_STORE_USER)) 0.00 : ffff800010252b18: ldr w0, [x21, #8] 0.00 : ffff800010252b1c: tbz w0, #16, ffff800010252ab4 <__slab_free+0x374> : __list_del_entry(): : __list_del(entry->prev, entry->next); 0.00 : ffff800010252b20: ldp x1, x0, [x19, #8] : __list_del(): : next->prev = prev; 0.00 : ffff800010252b24: str x0, [x1, #8] : list_del(): : entry->next = LIST_POISON1; 0.00 : ffff800010252b28: mov x3, #0x100 // #256 : entry->prev = LIST_POISON2; 0.00 : ffff800010252b2c: mov x2, #0x122 // #290 : entry->next = LIST_POISON1; 0.00 : ffff800010252b30: movk x3, #0xdead, lsl #48 : entry->prev = LIST_POISON2; 0.00 : ffff800010252b34: movk x2, #0xdead, lsl #48 : __write_once_size(): 0.00 : ffff800010252b38: str x1, [x0] : list_del(): 0.00 : ffff800010252b3c: stp x3, x2, [x19, #8] 0.00 : ffff800010252b40: b ffff800010252ab4 <__slab_free+0x374> : __list_del_entry(): : __list_del(entry->prev, entry->next); 0.00 : ffff800010252b44: ldp x1, x0, [x19, #8] : __list_del(): : next->prev = prev; 0.00 : ffff800010252b48: str x0, [x1, #8] : list_del(): : entry->next = LIST_POISON1; 0.00 : ffff800010252b4c: mov x3, #0x100 // #256 : entry->prev = LIST_POISON2; 0.00 : ffff800010252b50: mov x2, #0x122 // #290 : entry->next = LIST_POISON1; 0.00 : ffff800010252b54: movk x3, #0xdead, lsl #48 : entry->prev = LIST_POISON2; 0.00 : ffff800010252b58: movk x2, #0xdead, lsl #48 : __write_once_size(): 0.00 : ffff800010252b5c: str x1, [x0] : list_del(): 0.00 : ffff800010252b60: stp x3, x2, [x19, #8] 0.00 : ffff800010252b64: b ffff800010252af0 <__slab_free+0x3b0> Percent | Source code & Disassembly of vmlinux for cycles (56478 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff8000107d2710 : : nvme_cqe_pending(): : } : : /* We read the CQE phase first to check if the rest of the entry is valid */ : static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq) : { : return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == 1.38 : ffff8000107d2710: ldrh w0, [x1, #112] 19.64 : ffff8000107d2714: ldr x2, [x1, #72] : nvme_irq_check(): : } : : static irqreturn_t nvme_irq_check(int irq, void *data) : { : struct nvme_queue *nvmeq = data; : if (nvme_cqe_pending(nvmeq)) 2.57 : ffff8000107d2718: ldrb w1, [x1, #116] : nvme_cqe_pending(): : return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == 0.01 : ffff8000107d271c: add x0, x2, x0, lsl #4 0.36 : ffff8000107d2720: ldrh w0, [x0, #14] : nvme_irq_check(): : if (nvme_cqe_pending(nvmeq)) 2.99 : ffff8000107d2724: and w0, w0, #0x1 : return IRQ_WAKE_THREAD; : return IRQ_NONE; 1.54 : ffff8000107d2728: cmp w1, w0 0.52 : ffff8000107d272c: cset w0, eq // eq = none : } 70.99 : ffff8000107d2730: lsl w0, w0, #1 0.00 : ffff8000107d2734: ret Percent | Source code & Disassembly of vmlinux for cycles (49244 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010465af8 : : blk_mq_free_request(): : blk_mq_sched_restart(hctx); : blk_queue_exit(q); : } : : void blk_mq_free_request(struct request *rq) : { 0.03 : ffff800010465af8: stp x29, x30, [sp, #-48]! 0.00 : ffff800010465afc: mov x29, sp 0.07 : ffff800010465b00: stp x19, x20, [sp, #16] 0.00 : ffff800010465b04: mov x19, x0 0.09 : ffff800010465b08: stp x21, x22, [sp, #32] : struct request_queue *q = rq->q; : struct elevator_queue *e = q->elevator; : struct blk_mq_ctx *ctx = rq->mq_ctx; : struct blk_mq_hw_ctx *hctx = rq->mq_hctx; : : if (rq->rq_flags & RQF_ELVPRIV) { 0.50 : ffff800010465b0c: ldr w1, [x0, #28] : struct blk_mq_ctx *ctx = rq->mq_ctx; 0.11 : ffff800010465b10: ldp x21, x22, [x0] : struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 0.12 : ffff800010465b14: ldr x20, [x0, #16] : if (rq->rq_flags & RQF_ELVPRIV) { 0.00 : ffff800010465b18: tbz w1, #12, ffff800010465b48 : struct elevator_queue *e = q->elevator; 0.00 : ffff800010465b1c: ldr x1, [x21, #8] : if (e && e->type->ops.finish_request) 0.00 : ffff800010465b20: cbz x1, ffff800010465b34 0.00 : ffff800010465b24: ldr x1, [x1] 0.00 : ffff800010465b28: ldr x1, [x1, #104] 0.00 : ffff800010465b2c: cbz x1, ffff800010465b34 : e->type->ops.finish_request(rq); 0.00 : ffff800010465b30: blr x1 : if (rq->elv.icq) { 0.00 : ffff800010465b34: ldr x0, [x19, #128] 0.00 : ffff800010465b38: cbz x0, ffff800010465b48 : put_io_context(rq->elv.icq->ioc); 0.00 : ffff800010465b3c: ldr x0, [x0, #8] 0.00 : ffff800010465b40: bl ffff80001045fe28 : rq->elv.icq = NULL; 0.00 : ffff800010465b44: str xzr, [x19, #128] : } : } : : ctx->rq_completed[rq_is_sync(rq)]++; 0.11 : ffff800010465b48: ldr w1, [x19, #24] : op_is_sync(): : * PREFLUSH flag. Other operations may be marked as synchronous using the : * REQ_SYNC flag. : */ : static inline bool op_is_sync(unsigned int op) : { : return (op & REQ_OP_MASK) == REQ_OP_READ || 0.00 : ffff800010465b4c: mov w0, #0x1 // #1 0.00 : ffff800010465b50: and w2, w1, #0xff 0.00 : ffff800010465b54: cbz w2, ffff800010465b68 0.00 : ffff800010465b58: and w1, w1, #0x7f800 0.00 : ffff800010465b5c: and w1, w1, #0xfffe0fff 0.00 : ffff800010465b60: cmp w1, #0x0 0.00 : ffff800010465b64: cset w0, ne // ne = any 0.51 : ffff800010465b68: sxtw x0, w0 0.00 : ffff800010465b6c: add x0, x22, x0, lsl #3 : blk_mq_free_request(): 0.08 : ffff800010465b70: ldr x1, [x0, #128] 0.00 : ffff800010465b74: add x1, x1, #0x1 14.51 : ffff800010465b78: str x1, [x0, #128] : if (rq->rq_flags & RQF_MQ_INFLIGHT) 0.12 : ffff800010465b7c: ldr w0, [x19, #28] 0.00 : ffff800010465b80: tbnz w0, #6, ffff800010465c0c : atomic_dec(&hctx->nr_active); : : if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) 0.31 : ffff800010465b84: adrp x0, ffff800011b26000 0.00 : ffff800010465b88: ldr w0, [x0, #2556] 0.00 : ffff800010465b8c: cbnz w0, ffff800010465c30 : laptop_io_completion(q->backing_dev_info); : : rq_qos_done(q, rq); 0.36 : ffff800010465b90: ldr x0, [x21, #24] : rq_qos_done(): : __rq_qos_cleanup(q->rq_qos, bio); : } : : static inline void rq_qos_done(struct request_queue *q, struct request *rq) : { : if (q->rq_qos) 0.00 : ffff800010465b94: cbz x0, ffff800010465ba0 : __rq_qos_done(q->rq_qos, rq); 0.00 : ffff800010465b98: mov x1, x19 0.00 : ffff800010465b9c: bl ffff800010476170 <__rq_qos_done> : blk_mq_free_request(): : : WRITE_ONCE(rq->state, MQ_RQ_IDLE); : if (refcount_dec_and_test(&rq->ref)) 0.08 : ffff800010465ba0: add x0, x19, #0xd4 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.36 : ffff800010465ba4: str wzr, [x19, #208] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010465ba8: b ffff800010465be0 0.00 : ffff800010465bac: b ffff800010465be0 : __lse_atomic_fetch_sub_release(): : return i; \ : } : : ATOMIC_FETCH_OP_SUB(_relaxed, ) : ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") : ATOMIC_FETCH_OP_SUB(_release, l, "memory") 0.26 : ffff800010465bb0: mov w1, #0x1 // #1 0.00 : ffff800010465bb4: neg w1, w1 0.08 : ffff800010465bb8: ldaddl w1, w1, [x0] : refcount_sub_and_test(): : */ : static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) : { : int old = atomic_fetch_sub_release(i, &r->refs); : : if (old == i) { 74.86 : ffff800010465bbc: cmp w1, #0x1 0.00 : ffff800010465bc0: b.ne ffff800010465bf4 // b.any : smp_acquire__after_ctrl_dep(); 0.01 : ffff800010465bc4: dmb ishld : blk_mq_free_request(): : __blk_mq_free_request(rq); 7.02 : ffff800010465bc8: mov x0, x19 0.00 : ffff800010465bcc: bl ffff800010464258 <__blk_mq_free_request> : } 0.10 : ffff800010465bd0: ldp x19, x20, [sp, #16] 0.15 : ffff800010465bd4: ldp x21, x22, [sp, #32] 0.15 : ffff800010465bd8: ldp x29, x30, [sp], #48 0.00 : ffff800010465bdc: ret : __ll_sc_atomic_fetch_sub_release(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010465be0: mov w2, #0x1 // #1 0.00 : ffff800010465be4: add x5, x19, #0xd4 0.00 : ffff800010465be8: b ffff800010469e68 : refcount_sub_and_test(): : if (old == i) { 0.00 : ffff800010465bec: cmp w1, #0x1 0.00 : ffff800010465bf0: b.eq ffff800010465bc4 // b.none : return true; : } : : if (unlikely(old < 0 || old - i < 0)) 0.00 : ffff800010465bf4: cmp w1, #0x0 0.00 : ffff800010465bf8: b.le ffff800010465c5c : blk_mq_free_request(): 0.00 : ffff800010465bfc: ldp x19, x20, [sp, #16] 0.00 : ffff800010465c00: ldp x21, x22, [sp, #32] 0.00 : ffff800010465c04: ldp x29, x30, [sp], #48 0.00 : ffff800010465c08: ret : atomic_dec(&hctx->nr_active); 0.00 : ffff800010465c0c: add x1, x20, #0x1b0 : arch_static_branch_jump(): 0.00 : ffff800010465c10: b ffff800010465c4c 0.00 : ffff800010465c14: b ffff800010465c4c : __lse_atomic_sub(): : asm volatile( 0.00 : ffff800010465c18: mov w0, #0x1 // #1 0.00 : ffff800010465c1c: neg w0, w0 0.00 : ffff800010465c20: stadd w0, [x1] : blk_mq_free_request(): : if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) 0.00 : ffff800010465c24: adrp x0, ffff800011b26000 0.00 : ffff800010465c28: ldr w0, [x0, #2556] 0.00 : ffff800010465c2c: cbz w0, ffff800010465b90 : blk_rq_is_scsi(): : return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT; : } : : static inline bool blk_rq_is_scsi(struct request *rq) : { : return blk_op_is_scsi(req_op(rq)); 0.00 : ffff800010465c30: ldrb w0, [x19, #24] : blk_rq_is_passthrough(): : return blk_op_is_private(req_op(rq)); : } : : static inline bool blk_rq_is_passthrough(struct request *rq) : { : return blk_rq_is_scsi(rq) || blk_rq_is_private(rq); 0.00 : ffff800010465c34: sub w0, w0, #0x20 0.00 : ffff800010465c38: cmp w0, #0x3 0.00 : ffff800010465c3c: b.ls ffff800010465b90 // b.plast : blk_mq_free_request(): : laptop_io_completion(q->backing_dev_info); 0.00 : ffff800010465c40: ldr x0, [x21, #88] 0.00 : ffff800010465c44: bl ffff8000101ddc80 0.00 : ffff800010465c48: b ffff800010465b90 : __ll_sc_atomic_sub(): 0.00 : ffff800010465c4c: mov w0, #0x1 // #1 0.00 : ffff800010465c50: add x3, x20, #0x1b0 0.00 : ffff800010465c54: b ffff800010469e80 0.00 : ffff800010465c58: b ffff800010465b84 : refcount_sub_and_test(): : refcount_warn_saturate(r, REFCOUNT_SUB_UAF); 0.00 : ffff800010465c5c: mov w1, #0x3 // #3 0.00 : ffff800010465c60: bl ffff8000104915e0 : blk_mq_free_request(): : } 0.00 : ffff800010465c64: ldp x19, x20, [sp, #16] 0.00 : ffff800010465c68: ldp x21, x22, [sp, #32] 0.00 : ffff800010465c6c: ldp x29, x30, [sp], #48 0.00 : ffff800010465c70: ret Percent | Source code & Disassembly of vmlinux for cycles (49430 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff8000102d8ec8 : : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000102d8ec8: mrs x6, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.01 : ffff8000102d8ecc: and w2, w6, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff8000102d8ed0: cbnz w2, ffff8000102d8edc : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff8000102d8ed4: mov x2, #0x60 // #96 0.00 : ffff8000102d8ed8: msr daifset, #0x2 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.20 : ffff8000102d8edc: mrs x5, tpidr_el1 : put_reqs_available(): : { : struct kioctx_cpu *kcpu; : unsigned long flags; : : local_irq_save(flags); : kcpu = this_cpu_ptr(ctx->cpu); 0.55 : ffff8000102d8ee0: ldr x2, [x0, #128] : kcpu->reqs_available += nr; : : while (kcpu->reqs_available >= ctx->req_batch * 2) { : kcpu->reqs_available -= ctx->req_batch; : atomic_add(ctx->req_batch, &ctx->reqs_available); 0.00 : ffff8000102d8ee4: add x3, x0, #0x100 : kcpu->reqs_available += nr; 0.00 : ffff8000102d8ee8: ldr w7, [x2, x5] : kcpu = this_cpu_ptr(ctx->cpu); 0.00 : ffff8000102d8eec: add x4, x2, x5 : kcpu->reqs_available += nr; 0.00 : ffff8000102d8ef0: add w1, w1, w7 0.08 : ffff8000102d8ef4: str w1, [x2, x5] 0.19 : ffff8000102d8ef8: ldr w2, [x0, #136] : kcpu->reqs_available -= ctx->req_batch; 0.00 : ffff8000102d8efc: sub w5, w1, w2 : while (kcpu->reqs_available >= ctx->req_batch * 2) { 0.00 : ffff8000102d8f00: cmp w1, w2, lsl #1 0.07 : ffff8000102d8f04: b.cc ffff8000102d8f34 // b.lo, b.ul, b.last : kcpu->reqs_available -= ctx->req_batch; 0.57 : ffff8000102d8f08: str w5, [x4] : atomic_add(ctx->req_batch, &ctx->reqs_available); 0.82 : ffff8000102d8f0c: ldr w2, [x0, #136] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102d8f10: b ffff8000102d8f40 0.03 : ffff8000102d8f14: b ffff8000102d8f40 : __lse_atomic_add(): : } : : ATOMIC_OP(andnot, stclr) : ATOMIC_OP(or, stset) : ATOMIC_OP(xor, steor) : ATOMIC_OP(add, stadd) 0.11 : ffff8000102d8f18: mov w1, w2 0.00 : ffff8000102d8f1c: add x5, x0, #0x100 0.11 : ffff8000102d8f20: stadd w1, [x5] 0.83 : ffff8000102d8f24: ldr w1, [x4] : put_reqs_available(): : kcpu->reqs_available -= ctx->req_batch; 0.00 : ffff8000102d8f28: sub w5, w1, w2 : while (kcpu->reqs_available >= ctx->req_batch * 2) { 0.00 : ffff8000102d8f2c: cmp w1, w2, lsl #1 0.00 : ffff8000102d8f30: b.cs ffff8000102d8f08 // b.hs, b.nlast : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.76 : ffff8000102d8f34: msr daif, x6 : arch_static_branch(): : asm_volatile_goto( 14.10 : ffff8000102d8f38: nop : put_reqs_available(): : } : : local_irq_restore(flags); : } 0.00 : ffff8000102d8f3c: ret : __ll_sc_atomic_add(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff8000102d8f40: b ffff8000102dd850 <__arm64_compat_sys_io_pgetevents_time64+0x1e0> 0.00 : ffff8000102d8f44: ldr w2, [x0, #136] 0.00 : ffff8000102d8f48: ldr w1, [x4] 0.00 : ffff8000102d8f4c: b ffff8000102d8f28 : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.63 : ffff8000102d8f50: dsb sy : put_reqs_available(): 80.94 : ffff8000102d8f54: ret Percent | Source code & Disassembly of vmlinux for cycles (50171 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010aeb340 : : __arch_counter_get_cntpct(): : : static __always_inline u64 __arch_counter_get_cntpct(void) : { : u64 cnt; : : isb(); 0.18 : ffff800010aeb340: isb : cnt = read_sysreg(cntpct_el0); 95.23 : ffff800010aeb344: mrs x0, cntpct_el0 : arch_counter_enforce_ordering(cnt); 4.60 : ffff800010aeb348: eor x1, x0, x0 0.00 : ffff800010aeb34c: add x1, sp, x1 0.00 : ffff800010aeb350: ldr xzr, [x1] : arch_counter_get_cntpct(): : } : : static notrace u64 arch_counter_get_cntpct(void) : { : return __arch_counter_get_cntpct(); : } 0.00 : ffff800010aeb354: ret Percent | Source code & Disassembly of vmlinux for cycles (49202 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010cd80f0 <_raw_spin_lock>: : _raw_spin_lock(): : EXPORT_SYMBOL(_raw_spin_trylock_bh); : #endif : : #ifndef CONFIG_INLINE_SPIN_LOCK : void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) : { 1.57 : ffff800010cd80f0: stp x29, x30, [sp, #-16]! 0.00 : ffff800010cd80f4: mov x3, x0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 1.07 : ffff800010cd80f8: mrs x2, sp_el0 : _raw_spin_lock(): 0.07 : ffff800010cd80fc: mov x29, sp : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.32 : ffff800010cd8100: ldr w1, [x2, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.01 : ffff800010cd8104: add w1, w1, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.52 : ffff800010cd8108: str w1, [x2, #16] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010cd810c: b ffff800010cd8134 <_raw_spin_lock+0x44> 0.58 : ffff800010cd8110: b ffff800010cd8134 <_raw_spin_lock+0x44> : __lse__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, ) : __CMPXCHG_CASE(w, , , 32, ) : __CMPXCHG_CASE(x, , , 64, ) : __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") : __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 0.10 : ffff800010cd8114: mov w1, #0x0 // #0 0.00 : ffff800010cd8118: mov w2, #0x1 // #1 0.27 : ffff800010cd811c: mov w4, w1 0.11 : ffff800010cd8120: casa w4, w2, [x0] 94.32 : ffff800010cd8124: mov w0, w4 : atomic_try_cmpxchg_acquire(): : static inline bool : atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) : { : int r, o = *old; : r = atomic_cmpxchg_acquire(v, o, new); : if (unlikely(r != o)) 0.00 : ffff800010cd8128: cbnz w0, ffff800010cd8144 <_raw_spin_lock+0x54> : _raw_spin_lock(): : __raw_spin_lock(lock); : } 0.02 : ffff800010cd812c: ldp x29, x30, [sp], #16 0.05 : ffff800010cd8130: ret : __ll_sc__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, , , , , K) : __CMPXCHG_CASE(w, , , 32, , , , , K) : __CMPXCHG_CASE( , , , 64, , , , , L) : __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) : __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 0.00 : ffff800010cd8134: mov x1, #0x0 // #0 0.00 : ffff800010cd8138: mov w2, #0x1 // #1 0.00 : ffff800010cd813c: b ffff800010cd880c <_raw_read_lock_irqsave+0x134> : atomic_try_cmpxchg_acquire(): 0.00 : ffff800010cd8140: cbz w0, ffff800010cd812c <_raw_spin_lock+0x3c> : queued_spin_lock(): : u32 val = 0; : : if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL))) : return; : : queued_spin_lock_slowpath(lock, val); 0.00 : ffff800010cd8144: mov w1, w0 0.00 : ffff800010cd8148: mov x0, x3 0.00 : ffff800010cd814c: bl ffff800010139e90 : _raw_spin_lock(): 0.00 : ffff800010cd8150: ldp x29, x30, [sp], #16 0.00 : ffff800010cd8154: ret Percent | Source code & Disassembly of vmlinux for cycles (46468 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010116ad0 : : try_to_wake_up(): : * Return: %true if @p->state changes (an actual wakeup was done), : * %false otherwise. : */ : static int : try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) : { 0.07 : ffff800010116ad0: stp x29, x30, [sp, #-112]! 0.00 : ffff800010116ad4: mov x29, sp 0.60 : ffff800010116ad8: stp x19, x20, [sp, #16] 0.00 : ffff800010116adc: adrp x20, ffff800011909000 0.02 : ffff800010116ae0: str x21, [sp, #32] 0.00 : ffff800010116ae4: add x3, x20, #0x908 0.00 : ffff800010116ae8: stp x24, x25, [sp, #56] 0.00 : ffff800010116aec: mov x19, x0 0.00 : ffff800010116af0: mov w21, w2 0.15 : ffff800010116af4: ldr x4, [x3] 0.81 : ffff800010116af8: str x4, [x29, #104] 0.00 : ffff800010116afc: mov x4, #0x0 // #0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010116b00: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010116b04: ldr w2, [x0, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010116b08: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.64 : ffff800010116b0c: str w2, [x0, #16] : try_to_wake_up(): : unsigned long flags; : int cpu, success = 0; : : preempt_disable(); 0.00 : ffff800010116b10: mov w24, w1 : if (p == current) { 0.00 : ffff800010116b14: cmp x19, x0 0.00 : ffff800010116b18: b.eq ffff800010116d64 // b.none 0.37 : ffff800010116b1c: stp x22, x23, [x29, #40] : * If we are going to wake up a thread waiting for CONDITION we : * need to ensure that CONDITION=1 done by the caller can not be : * reordered with p->state check below. This pairs with mb() in : * set_current_state() the waiting thread does. : */ : raw_spin_lock_irqsave(&p->pi_lock, flags); 0.15 : ffff800010116b20: add x22, x19, #0x6f4 0.00 : ffff800010116b24: mov x0, x22 0.00 : ffff800010116b28: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.06 : ffff800010116b2c: mov x23, x0 : smp_mb__after_spinlock(); 0.03 : ffff800010116b30: dmb ish : if (!(p->state & state)) 5.37 : ffff800010116b34: ldr x0, [x19, #24] : int cpu, success = 0; 0.00 : ffff800010116b38: mov w25, #0x0 // #0 : if (!(p->state & state)) 0.00 : ffff800010116b3c: tst x0, x24 0.01 : ffff800010116b40: b.ne ffff800010116bb0 // b.any : : #endif /* CONFIG_SMP */ : : ttwu_queue(p, cpu, wake_flags); : unlock: : raw_spin_unlock_irqrestore(&p->pi_lock, flags); 0.61 : ffff800010116b44: mov x1, x23 0.00 : ffff800010116b48: mov x0, x22 0.00 : ffff800010116b4c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.01 : ffff800010116b50: ldp x22, x23, [x29, #40] : get_current(): 2.06 : ffff800010116b54: mrs x1, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.03 : ffff800010116b58: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010116b5c: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.60 : ffff800010116b60: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010116b64: cbz x0, ffff800010116b9c : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010116b68: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010116b6c: cbz x0, ffff800010116b9c : try_to_wake_up(): : if (success) : ttwu_stat(p, cpu, wake_flags); : preempt_enable(); : : return success; : } 1.10 : ffff800010116b70: add x20, x20, #0x908 0.00 : ffff800010116b74: mov w0, w25 0.08 : ffff800010116b78: ldr x2, [x29, #104] 0.00 : ffff800010116b7c: ldr x1, [x20] 0.00 : ffff800010116b80: eor x1, x2, x1 0.00 : ffff800010116b84: cbnz x1, ffff800010116eec 0.01 : ffff800010116b88: ldp x19, x20, [sp, #16] 0.02 : ffff800010116b8c: ldr x21, [sp, #32] 0.09 : ffff800010116b90: ldp x24, x25, [sp, #56] 3.40 : ffff800010116b94: ldp x29, x30, [sp], #112 0.00 : ffff800010116b98: ret : get_current(): 0.00 : ffff800010116b9c: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff800010116ba0: ldr w0, [x0, #16] : preempt_schedule(): : { : /* : * If there is a non-zero preempt_count or interrupts are disabled, : * we do not want to preempt the current task. Just return.. : */ : if (likely(!preemptible())) 0.00 : ffff800010116ba4: cbnz w0, ffff800010116b70 0.00 : ffff800010116ba8: bl ffff800010cd2e28 : try_to_wake_up(): : return success; 0.00 : ffff800010116bac: b ffff800010116b70 : __read_once_size(): 36.75 : ffff800010116bb0: ldr w0, [x19, #68] : try_to_wake_up(): : smp_rmb(); 9.05 : ffff800010116bb4: dmb ishld : if (p->on_rq && ttwu_remote(p, wake_flags)) 4.80 : ffff800010116bb8: ldr w0, [x19, #104] 0.00 : ffff800010116bbc: cbnz w0, ffff800010116d84 0.04 : ffff800010116bc0: stp x26, x27, [x29, #72] 0.05 : ffff800010116bc4: str x28, [x29, #88] : smp_rmb(); 0.00 : ffff800010116bc8: dmb ishld : smp_cond_load_acquire(&p->on_cpu, !VAL); 6.96 : ffff800010116bcc: add x1, x19, #0x40 0.01 : ffff800010116bd0: ldar w0, [x1] 0.00 : ffff800010116bd4: cbz w0, ffff800010116bfc 0.44 : ffff800010116bd8: sxtw x0, w0 : __cmpwait_case_32(): : : [val] "r" (val)); \ : } : : __CMPWAIT_CASE(w, b, 8); : __CMPWAIT_CASE(w, h, 16); : __CMPWAIT_CASE(w, , 32); 0.00 : ffff800010116bdc: sevl 0.03 : ffff800010116be0: wfe 1.63 : ffff800010116be4: ldxr w2, [x1] 0.00 : ffff800010116be8: eor w2, w2, w0 0.00 : ffff800010116bec: cbnz w2, ffff800010116bf4 0.64 : ffff800010116bf0: wfe : try_to_wake_up(): 5.07 : ffff800010116bf4: ldar w0, [x1] 0.00 : ffff800010116bf8: cbnz w0, ffff800010116bd8 : p->sched_contributes_to_load = !!task_contributes_to_load(p); 1.19 : ffff800010116bfc: ldr x1, [x19, #24] 0.57 : ffff800010116c00: mov w0, #0x0 // #0 0.00 : ffff800010116c04: tbz w1, #1, ffff800010116c20 0.00 : ffff800010116c08: ldr w1, [x19, #44] 0.00 : ffff800010116c0c: tbnz w1, #16, ffff800010116c20 0.00 : ffff800010116c10: ldr x0, [x19, #24] 0.00 : ffff800010116c14: lsr x0, x0, #10 0.00 : ffff800010116c18: eor x0, x0, #0x1 0.00 : ffff800010116c1c: and w0, w0, #0x1 0.04 : ffff800010116c20: ldrb w1, [x19, #1060] : p->state = TASK_WAKING; 0.00 : ffff800010116c24: mov x3, #0x200 // #512 : if (p->in_iowait) { 0.41 : ffff800010116c28: ldrb w2, [x19, #1064] 0.00 : ffff800010116c2c: adrp x24, ffff800011528000 : p->state = TASK_WAKING; 0.00 : ffff800010116c30: str x3, [x19, #24] 0.00 : ffff800010116c34: adrp x27, ffff800011909000 : p->sched_contributes_to_load = !!task_contributes_to_load(p); 0.00 : ffff800010116c38: bfi w1, w0, #1, #1 0.01 : ffff800010116c3c: strb w1, [x19, #1060] : if (p->in_iowait) { 0.00 : ffff800010116c40: tbnz w2, #1, ffff800010116dec : select_task_rq(): : if (p->nr_cpus_allowed > 1) 0.01 : ffff800010116c44: ldr w0, [x19, #732] : try_to_wake_up(): : cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); 0.53 : ffff800010116c48: ldr w1, [x19, #100] : select_task_rq(): : if (p->nr_cpus_allowed > 1) 0.00 : ffff800010116c4c: cmp w0, #0x1 0.00 : ffff800010116c50: b.le ffff800010116ed4 : cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); 0.09 : ffff800010116c54: ldr x4, [x19, #128] 0.00 : ffff800010116c58: mov w3, w21 0.00 : ffff800010116c5c: mov w2, #0x10 // #16 0.00 : ffff800010116c60: mov x0, x19 0.20 : ffff800010116c64: ldr x4, [x4, #80] 0.00 : ffff800010116c68: blr x4 0.26 : ffff800010116c6c: mov w26, w0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.02 : ffff800010116c70: add w0, w26, #0x3f 0.00 : ffff800010116c74: cmp w26, #0x0 0.01 : ffff800010116c78: csel w0, w0, w26, lt // lt = tstop 0.08 : ffff800010116c7c: ldr x1, [x19, #736] 0.00 : ffff800010116c80: and w2, w26, #0x3f 0.00 : ffff800010116c84: asr w0, w0, #6 0.00 : ffff800010116c88: sxtw x0, w0 0.18 : ffff800010116c8c: ldr x1, [x1, x0, lsl #3] 0.00 : ffff800010116c90: lsr x1, x1, x2 : is_cpu_allowed(): : if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 0.00 : ffff800010116c94: tbz w1, #0, ffff800010116eb8 : is_per_cpu_kthread(): : #endif : : #ifdef CONFIG_SMP : static inline bool is_per_cpu_kthread(struct task_struct *p) : { : if (!(p->flags & PF_KTHREAD)) 0.27 : ffff800010116c98: ldr w1, [x19, #44] 0.01 : ffff800010116c9c: tbz w1, #21, ffff800010116cac : return false; : : if (p->nr_cpus_allowed != 1) 0.00 : ffff800010116ca0: ldr w1, [x19, #732] 0.00 : ffff800010116ca4: cmp w1, #0x1 0.00 : ffff800010116ca8: b.eq ffff800010116e70 // b.none : test_bit(): 0.23 : ffff800010116cac: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010116cb0: add x1, x1, #0x1c8 0.06 : ffff800010116cb4: ldr x0, [x1, x0, lsl #3] 0.05 : ffff800010116cb8: lsr x0, x0, x2 : is_cpu_allowed(): : return cpu_active(cpu); 0.01 : ffff800010116cbc: and w0, w0, #0x1 : select_task_rq(): : if (unlikely(!is_cpu_allowed(p, cpu))) 0.01 : ffff800010116cc0: cbz w0, ffff800010116eb8 : __read_once_size(): 0.19 : ffff800010116cc4: ldr w0, [x19, #68] : try_to_wake_up(): : if (task_cpu(p) != cpu) { 0.03 : ffff800010116cc8: cmp w26, w0 0.01 : ffff800010116ccc: b.eq ffff800010116ce0 // b.none : wake_flags |= WF_MIGRATED; 0.00 : ffff800010116cd0: orr w21, w21, #0x4 : set_task_cpu(p, cpu); 0.00 : ffff800010116cd4: mov w1, w26 0.00 : ffff800010116cd8: mov x0, x19 0.00 : ffff800010116cdc: bl ffff800010116390 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.26 : ffff800010116ce0: mrs x1, tpidr_el1 : ttwu_queue(): : if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { 0.21 : ffff800010116ce4: adrp x0, ffff80001151d000 0.00 : ffff800010116ce8: add x0, x0, #0x18 : cpus_share_cache(): : return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 0.05 : ffff800010116cec: ldrsw x2, [x0, x1] : ttwu_queue(): : struct rq *rq = cpu_rq(cpu); 0.00 : ffff800010116cf0: add x27, x27, #0x928 0.01 : ffff800010116cf4: sxtw x28, w26 : cpus_share_cache(): : return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 0.01 : ffff800010116cf8: adrp x0, ffff80001151f000 0.11 : ffff800010116cfc: add x0, x0, #0x124 : ttwu_queue(): : struct rq *rq = cpu_rq(cpu); 0.12 : ffff800010116d00: add x24, x24, #0xe80 : cpus_share_cache(): : return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 0.17 : ffff800010116d04: ldr x3, [x27, x2, lsl #3] : ttwu_queue(): : struct rq *rq = cpu_rq(cpu); 0.28 : ffff800010116d08: ldr x1, [x27, x28, lsl #3] : if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { 0.10 : ffff800010116d0c: ldr w2, [x1, x0] 10.12 : ffff800010116d10: ldr w0, [x3, x0] 0.00 : ffff800010116d14: cmp w0, w2 0.00 : ffff800010116d18: b.eq ffff800010116db0 // b.none : sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 0.00 : ffff800010116d1c: mov w0, w26 0.00 : ffff800010116d20: bl ffff80001011b140 : ttwu_queue_remote(): : p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 0.00 : ffff800010116d24: ldrb w2, [x19, #1060] 0.00 : ffff800010116d28: ubfx x21, x21, #2, #1 : if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { 0.00 : ffff800010116d2c: add x1, x19, #0x38 : try_to_wake_up(): : success = 1; 0.00 : ffff800010116d30: mov w25, #0x1 // #1 : llist_add(): : * : * Returns true if the list was empty prior to adding this entry. : */ : static inline bool llist_add(struct llist_node *new, struct llist_head *head) : { : return llist_add_batch(new, new, head); 0.00 : ffff800010116d34: mov x0, x1 : ttwu_queue_remote(): : p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 0.00 : ffff800010116d38: bfi w2, w21, #3, #1 0.00 : ffff800010116d3c: strb w2, [x19, #1060] : if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { 0.00 : ffff800010116d40: ldr x2, [x27, x28, lsl #3] 0.00 : ffff800010116d44: add x2, x24, x2 : llist_add(): 0.00 : ffff800010116d48: add x2, x2, #0xba0 0.00 : ffff800010116d4c: bl ffff80001048db58 : ttwu_queue_remote(): 0.00 : ffff800010116d50: tst w0, #0xff 0.00 : ffff800010116d54: b.ne ffff800010116e5c // b.any 0.00 : ffff800010116d58: ldp x26, x27, [x29, #72] 0.00 : ffff800010116d5c: ldr x28, [x29, #88] 0.00 : ffff800010116d60: b ffff800010116b44 : try_to_wake_up(): : if (!(p->state & state)) 0.00 : ffff800010116d64: ldr x0, [x19, #24] : int cpu, success = 0; 0.00 : ffff800010116d68: mov w25, #0x0 // #0 : if (!(p->state & state)) 0.00 : ffff800010116d6c: tst x0, x24 0.00 : ffff800010116d70: b.eq ffff800010116b54 // b.none : __read_once_size(): 0.00 : ffff800010116d74: ldr w0, [x19, #68] : try_to_wake_up(): : success = 1; 0.00 : ffff800010116d78: mov w25, #0x1 // #1 : p->state = TASK_RUNNING; 0.01 : ffff800010116d7c: str xzr, [x19, #24] 0.00 : ffff800010116d80: b ffff800010116b54 : ttwu_remote(): : rq = __task_rq_lock(p, &rf); 0.01 : ffff800010116d84: add x1, x29, #0x60 0.00 : ffff800010116d88: mov x0, x19 0.00 : ffff800010116d8c: bl ffff800010114450 <__task_rq_lock> : task_on_rq_queued(): : return p->on_rq == TASK_ON_RQ_QUEUED; 0.00 : ffff800010116d90: ldr w25, [x19, #104] : ttwu_remote(): 0.00 : ffff800010116d94: mov x24, x0 : if (task_on_rq_queued(p)) { 0.00 : ffff800010116d98: cmp w25, #0x1 0.00 : ffff800010116d9c: b.eq ffff800010116e88 // b.none 0.00 : ffff800010116da0: stp x26, x27, [x29, #72] 0.00 : ffff800010116da4: str x28, [x29, #88] : __task_rq_unlock(): : raw_spin_unlock(&rq->lock); 0.00 : ffff800010116da8: bl ffff800010cd7c38 <_raw_spin_unlock> 0.00 : ffff800010116dac: b ffff800010116bc8 : ttwu_queue(): : struct rq *rq = cpu_rq(cpu); 0.45 : ffff800010116db0: add x24, x24, x1 : rq_lock(): : raw_spin_lock(&rq->lock); 0.26 : ffff800010116db4: mov x0, x24 0.04 : ffff800010116db8: bl ffff800010cd80f0 <_raw_spin_lock> : update_rq_clock(): : if (rq->clock_update_flags & RQCF_ACT_SKIP) 0.56 : ffff800010116dbc: ldr w0, [x24, #2392] 0.00 : ffff800010116dc0: tbz w0, #1, ffff800010116eac : ttwu_queue(): : ttwu_do_activate(rq, p, wake_flags, &rf); 0.00 : ffff800010116dc4: mov w2, w21 0.00 : ffff800010116dc8: mov x1, x19 0.00 : ffff800010116dcc: mov x0, x24 0.00 : ffff800010116dd0: bl ffff800010115f40 : rq_unlock(): : raw_spin_unlock(&rq->lock); 0.01 : ffff800010116dd4: mov x0, x24 : try_to_wake_up(): : success = 1; 0.00 : ffff800010116dd8: mov w25, #0x1 // #1 : rq_unlock(): 0.00 : ffff800010116ddc: bl ffff800010cd7c38 <_raw_spin_unlock> 0.45 : ffff800010116de0: ldr x28, [x29, #88] 0.37 : ffff800010116de4: ldp x26, x27, [x29, #72] 0.00 : ffff800010116de8: b ffff800010116b44 : delayacct_blkio_end(): : __delayacct_blkio_start(); : } : : static inline void delayacct_blkio_end(struct task_struct *p) : { : if (p->delays) 0.00 : ffff800010116dec: ldr x0, [x19, #2368] 0.00 : ffff800010116df0: cbz x0, ffff800010116dfc : __delayacct_blkio_end(p); 0.00 : ffff800010116df4: mov x0, x19 0.00 : ffff800010116df8: bl ffff8000101b2c58 <__delayacct_blkio_end> : get_current(): 0.00 : ffff800010116dfc: mrs x0, sp_el0 : delayacct_clear_flag(): : if (current->delays) 0.00 : ffff800010116e00: ldr x0, [x0, #2368] 0.00 : ffff800010116e04: cbz x0, ffff800010116e14 : current->delays->flags &= ~flag; 0.00 : ffff800010116e08: ldr w1, [x0, #4] 0.00 : ffff800010116e0c: and w1, w1, #0xfffffffd 0.00 : ffff800010116e10: str w1, [x0, #4] : __read_once_size(): 0.00 : ffff800010116e14: ldr w2, [x19, #68] : try_to_wake_up(): : atomic_dec(&task_rq(p)->nr_iowait); 0.00 : ffff800010116e18: adrp x27, ffff800011909000 0.00 : ffff800010116e1c: add x1, x27, #0x928 0.00 : ffff800010116e20: adrp x24, ffff800011528000 0.00 : ffff800010116e24: add x0, x24, #0xe80 0.00 : ffff800010116e28: ldr x1, [x1, w2, uxtw #3] 0.00 : ffff800010116e2c: add x0, x0, x1 0.00 : ffff800010116e30: add x2, x0, #0x998 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010116e34: b ffff800010116e4c 0.00 : ffff800010116e38: b ffff800010116e4c : __lse_atomic_sub(): : : #undef ATOMIC_FETCH_OP_AND : : static inline void __lse_atomic_sub(int i, atomic_t *v) : { : asm volatile( 0.00 : ffff800010116e3c: mov w1, #0x1 // #1 0.00 : ffff800010116e40: neg w1, w1 0.00 : ffff800010116e44: stadd w1, [x2] 0.00 : ffff800010116e48: b ffff800010116c44 : __ll_sc_atomic_sub(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010116e4c: mov w1, #0x1 // #1 0.00 : ffff800010116e50: add x0, x0, #0x998 0.00 : ffff800010116e54: b ffff80001011ab2c 0.00 : ffff800010116e58: b ffff800010116c44 : ttwu_queue_remote(): : smp_send_reschedule(cpu); 0.00 : ffff800010116e5c: mov w0, w26 0.00 : ffff800010116e60: bl ffff800010096ae0 0.00 : ffff800010116e64: ldp x26, x27, [x29, #72] 0.00 : ffff800010116e68: ldr x28, [x29, #88] 0.00 : ffff800010116e6c: b ffff800010116b44 : test_bit(): 0.00 : ffff800010116e70: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010116e74: add x1, x1, #0x160 0.01 : ffff800010116e78: ldr x1, [x1, x0, lsl #3] 0.00 : ffff800010116e7c: lsr x0, x1, x2 : is_cpu_allowed(): : return cpu_online(cpu); 0.00 : ffff800010116e80: and w0, w0, #0x1 0.00 : ffff800010116e84: b ffff800010116cc0 : update_rq_clock(): : if (rq->clock_update_flags & RQCF_ACT_SKIP) 0.00 : ffff800010116e88: ldr w1, [x0, #2392] 0.00 : ffff800010116e8c: tbz w1, #1, ffff800010116ecc : ttwu_remote(): : ttwu_do_wakeup(rq, p, wake_flags, &rf); 0.00 : ffff800010116e90: mov w2, w21 0.00 : ffff800010116e94: mov x1, x19 0.00 : ffff800010116e98: mov x0, x24 0.00 : ffff800010116e9c: bl ffff800010115ec0 : __task_rq_unlock(): : raw_spin_unlock(&rq->lock); 0.00 : ffff800010116ea0: mov x0, x24 0.00 : ffff800010116ea4: bl ffff800010cd7c38 <_raw_spin_unlock> 0.00 : ffff800010116ea8: b ffff800010116b44 : update_rq_clock(): 0.07 : ffff800010116eac: mov x0, x24 0.00 : ffff800010116eb0: bl ffff800010113d48 0.00 : ffff800010116eb4: b ffff800010116dc4 : __read_once_size(): 0.00 : ffff800010116eb8: ldr w0, [x19, #68] : select_task_rq(): : cpu = select_fallback_rq(task_cpu(p), p); 0.00 : ffff800010116ebc: mov x1, x19 0.00 : ffff800010116ec0: bl ffff800010116178 0.00 : ffff800010116ec4: mov w26, w0 0.00 : ffff800010116ec8: b ffff800010116cc4 : update_rq_clock(): 0.01 : ffff800010116ecc: bl ffff800010113d48 0.00 : ffff800010116ed0: b ffff800010116e90 : cpumask_first(): : * : * Returns >= nr_cpu_ids if no cpus set. : */ : static inline unsigned int cpumask_first(const struct cpumask *srcp) : { : return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits); 0.00 : ffff800010116ed4: ldr x0, [x19, #736] 0.00 : ffff800010116ed8: mov x2, #0x0 // #0 0.00 : ffff800010116edc: mov x1, #0x100 // #256 0.00 : ffff800010116ee0: bl ffff80001048daa8 : select_task_rq(): : cpu = cpumask_any(p->cpus_ptr); 0.00 : ffff800010116ee4: mov w26, w0 0.00 : ffff800010116ee8: b ffff800010116c70 0.00 : ffff800010116eec: stp x22, x23, [x29, #40] 0.00 : ffff800010116ef0: stp x26, x27, [x29, #72] 0.00 : ffff800010116ef4: str x28, [x29, #88] : try_to_wake_up(): : } 0.00 : ffff800010116ef8: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (45530 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010121cf0 : : select_task_rq_fair(): : * : * preempt must be disabled. : */ : static int : select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) : { 0.08 : ffff800010121cf0: stp x29, x30, [sp, #-208]! : struct sched_domain *tmp, *sd = NULL; : int cpu = smp_processor_id(); 0.01 : ffff800010121cf4: adrp x5, ffff80001151d000 0.00 : ffff800010121cf8: add x4, x5, #0x18 : { 0.00 : ffff800010121cfc: mov x29, sp 0.49 : ffff800010121d00: stp x19, x20, [sp, #16] 0.00 : ffff800010121d04: mov x20, x0 0.02 : ffff800010121d08: stp x21, x22, [sp, #32] : int new_cpu = prev_cpu; : int want_affine = 0; : int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 0.00 : ffff800010121d0c: ands w0, w3, #0x1 : { 0.01 : ffff800010121d10: stp x23, x24, [sp, #48] : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff800010121d14: mov x6, sp : select_task_rq_fair(): 0.04 : ffff800010121d18: str x25, [sp, #64] 0.00 : ffff800010121d1c: mov w21, w1 : int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 0.45 : ffff800010121d20: str w0, [x29, #168] : { 0.00 : ffff800010121d24: mov w19, w2 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.02 : ffff800010121d28: mrs x7, tpidr_el1 : select_task_rq_fair(): : int cpu = smp_processor_id(); 0.01 : ffff800010121d2c: ldr w24, [x4, x7] : int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 0.00 : ffff800010121d30: b.eq ffff800010121d48 // b.none : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010121d34: mrs x0, sp_el0 : select_task_rq_fair(): 0.00 : ffff800010121d38: ldr w0, [x0, #44] 0.00 : ffff800010121d3c: eor x0, x0, #0x4 0.00 : ffff800010121d40: ubfx x0, x0, #2, #1 0.00 : ffff800010121d44: str x0, [x29, #168] : : if (sd_flag & SD_BALANCE_WAKE) { 0.07 : ffff800010121d48: and w0, w19, #0x10 0.50 : ffff800010121d4c: str w0, [x29, #180] 0.00 : ffff800010121d50: tbz w19, #4, ffff8000101220c8 : record_wakee(): : if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 0.02 : ffff800010121d54: adrp x2, ffff800011907000 0.00 : ffff800010121d58: mov x0, #0xfa // #250 : get_current(): 0.01 : ffff800010121d5c: mrs x1, sp_el0 : record_wakee(): 0.06 : ffff800010121d60: ldr x4, [x2, #2432] 0.45 : ffff800010121d64: ldr x3, [x1, #80] 0.00 : ffff800010121d68: sub x0, x0, x4 0.00 : ffff800010121d6c: cmn x0, x3 0.00 : ffff800010121d70: b.pl ffff800010121d88 // b.nfrst : current->wakee_flips >>= 1; 0.00 : ffff800010121d74: ldr w0, [x1, #72] : current->wakee_flip_decay_ts = jiffies; 0.00 : ffff800010121d78: ldr x2, [x2, #2432] 0.00 : ffff800010121d7c: str x2, [x1, #80] : current->wakee_flips >>= 1; 0.00 : ffff800010121d80: lsr w0, w0, #1 0.00 : ffff800010121d84: str w0, [x1, #72] : get_current(): 0.02 : ffff800010121d88: mrs x0, sp_el0 : record_wakee(): : if (current->last_wakee != p) { 0.02 : ffff800010121d8c: ldr x1, [x0, #88] 0.00 : ffff800010121d90: cmp x20, x1 0.00 : ffff800010121d94: b.eq ffff800010121da8 // b.none : current->wakee_flips++; 0.02 : ffff800010121d98: ldr w1, [x0, #72] : current->last_wakee = p; 0.27 : ffff800010121d9c: str x20, [x0, #88] : current->wakee_flips++; 0.00 : ffff800010121da0: add w1, w1, #0x1 0.00 : ffff800010121da4: str w1, [x0, #72] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.02 : ffff800010121da8: nop 0.15 : ffff800010121dac: adrp x0, ffff800011528000 0.07 : ffff800010121db0: str x0, [x29, #184] 0.00 : ffff800010121db4: adrp x0, ffff800011909000 0.27 : ffff800010121db8: str x0, [x29, #200] : get_current(): 0.03 : ffff800010121dbc: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.20 : ffff800010121dc0: ldr w0, [x1, #16] : wake_wide(): : unsigned int slave = p->wakee_flips; 0.06 : ffff800010121dc4: ldr w23, [x20, #72] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010121dc8: add w0, w0, #0x1 : wake_wide(): : unsigned int master = current->wakee_flips; 2.67 : ffff800010121dcc: ldr w22, [x1, #72] : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 2.57 : ffff800010121dd0: str w0, [x1, #16] : wake_wide(): : int factor = this_cpu_read(sd_llc_size); 0.00 : ffff800010121dd4: adrp x0, ffff80001151f000 0.09 : ffff800010121dd8: add x0, x0, #0x120 : __my_cpu_offset(): 0.06 : ffff800010121ddc: mrs x2, tpidr_el1 : __read_once_size(): : __READ_ONCE_SIZE; 0.04 : ffff800010121de0: ldr w25, [x0, x2] 3.11 : ffff800010121de4: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010121de8: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 2.28 : ffff800010121dec: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010121df0: cbz x0, ffff800010121f9c : __read_once_size(): : __READ_ONCE_SIZE; 1.49 : ffff800010121df4: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.06 : ffff800010121df8: cbz x0, ffff800010121f9c : wake_wide(): : if (master < slave) 5.44 : ffff800010121dfc: cmp w22, w23 0.00 : ffff800010121e00: csel w0, w23, w22, cc // cc = lo, ul, last 0.00 : ffff800010121e04: csel w22, w22, w23, cc // cc = lo, ul, last : if (slave < factor || master < slave * factor) 0.00 : ffff800010121e08: cmp w25, w22 0.52 : ffff800010121e0c: b.hi ffff800010121fb4 // b.pmore 0.00 : ffff800010121e10: mul w22, w25, w22 0.00 : ffff800010121e14: sxtw x25, w24 0.00 : ffff800010121e18: cmp w22, w0 0.00 : ffff800010121e1c: b.hi ffff800010121fb4 // b.pmore : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff800010121e20: bl ffff80001015c1b0 <__rcu_read_lock> : select_task_rq_fair(): : int want_affine = 0; 0.00 : ffff800010121e24: mov w23, #0x0 // #0 : want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu) && : cpumask_test_cpu(cpu, p->cpus_ptr); : } : : rcu_read_lock(); : for_each_domain(cpu, tmp) { 0.00 : ffff800010121e28: ldr x0, [x29, #200] 0.00 : ffff800010121e2c: add x1, x0, #0x928 0.00 : ffff800010121e30: ldr x0, [x29, #184] 0.00 : ffff800010121e34: add x0, x0, #0xe80 0.00 : ffff800010121e38: ldr x1, [x1, x25, lsl #3] 0.00 : ffff800010121e3c: add x0, x0, x1 : __read_once_size(): 0.00 : ffff800010121e40: ldr x22, [x0, #2472] : select_task_rq_fair(): 0.00 : ffff800010121e44: cbz x22, ffff800010121eb0 : if (!(tmp->flags & SD_LOAD_BALANCE)) 4.21 : ffff800010121e48: ldr w0, [x22, #56] 0.00 : ffff800010121e4c: tbz w0, #0, ffff800010121eb0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 9.33 : ffff800010121e50: add w1, w21, #0x3f 0.00 : ffff800010121e54: cmp w21, #0x0 0.00 : ffff800010121e58: csel w1, w1, w21, lt // lt = tstop 0.00 : ffff800010121e5c: mov x4, #0x1 // #1 0.48 : ffff800010121e60: str x27, [x29, #80] 0.00 : ffff800010121e64: lsl x4, x4, x21 0.00 : ffff800010121e68: asr w1, w1, #6 : select_task_rq_fair(): : struct sched_domain *tmp, *sd = NULL; 0.00 : ffff800010121e6c: mov x27, #0x0 // #0 : test_bit(): 0.04 : ffff800010121e70: sbfiz x1, x1, #3, #32 0.00 : ffff800010121e74: nop : select_task_rq_fair(): : : /* : * If both 'cpu' and 'prev_cpu' are part of this domain, : * cpu is a valid SD_WAKE_AFFINE target. : */ : if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 0.04 : ffff800010121e78: cbz w23, ffff80001012202c : cpumask_test_cpu(): : * : * Returns 1 if @cpu is set in @cpumask, else returns 0 : */ : static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask) : { : return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); 0.00 : ffff800010121e7c: add x2, x22, #0x88 : select_task_rq_fair(): 0.00 : ffff800010121e80: tbz w0, #5, ffff800010121e90 : test_bit(): 0.53 : ffff800010121e84: ldr x2, [x2, x1] : select_task_rq_fair(): 0.00 : ffff800010121e88: tst x2, x4 0.00 : ffff800010121e8c: b.ne ffff800010122254 // b.any 0.00 : ffff800010121e90: tst w19, w0 0.00 : ffff800010121e94: csel x27, x27, x22, eq // eq = none : for_each_domain(cpu, tmp) { 0.00 : ffff800010121e98: ldr x22, [x22] 0.00 : ffff800010121e9c: cbz x22, ffff800010121ea8 : if (!(tmp->flags & SD_LOAD_BALANCE)) 0.00 : ffff800010121ea0: ldr w0, [x22, #56] 0.00 : ffff800010121ea4: tbnz w0, #0, ffff800010121e78 : sd = tmp; : else if (!want_affine) : break; : } : : if (unlikely(sd)) { 0.00 : ffff800010121ea8: cbnz x27, ffff8000101220e0 0.00 : ffff800010121eac: ldr x27, [x29, #80] : /* Slow path */ : new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag); : } else if (sd_flag & SD_BALANCE_WAKE) { /* XXX always ? */ 0.00 : ffff800010121eb0: ldr w0, [x29, #180] 0.00 : ffff800010121eb4: cbnz w0, ffff800010122008 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.04 : ffff800010121eb8: bl ffff800010160d20 <__rcu_read_unlock> : select_task_rq_fair(): : if (want_affine) : current->recent_used_cpu = cpu; : } : rcu_read_unlock(); : : return new_cpu; 0.04 : ffff800010121ebc: str w21, [x29, #192] : } 0.46 : ffff800010121ec0: ldr w0, [x29, #192] 0.46 : ffff800010121ec4: ldp x19, x20, [sp, #16] 0.18 : ffff800010121ec8: ldp x21, x22, [sp, #32] 0.05 : ffff800010121ecc: ldp x23, x24, [sp, #48] 0.15 : ffff800010121ed0: ldr x25, [sp, #64] 0.23 : ffff800010121ed4: ldp x29, x30, [sp], #208 0.00 : ffff800010121ed8: ret : find_energy_efficient_cpu(): : struct root_domain *rd = cpu_rq(smp_processor_id())->rd; 0.00 : ffff800010121edc: adrp x0, ffff800011909000 0.00 : ffff800010121ee0: add x5, x5, #0x18 0.00 : ffff800010121ee4: add x1, x0, #0x928 0.00 : ffff800010121ee8: str x0, [x29, #200] : __my_cpu_offset(): 0.00 : ffff800010121eec: mrs x0, tpidr_el1 : find_energy_efficient_cpu(): 0.00 : ffff800010121ef0: ldrsw x2, [x5, x0] 0.00 : ffff800010121ef4: adrp x0, ffff800011528000 0.00 : ffff800010121ef8: str x0, [x29, #184] 0.00 : ffff800010121efc: add x0, x0, #0xe80 0.00 : ffff800010121f00: ldr x1, [x1, x2, lsl #3] 0.00 : ffff800010121f04: add x0, x1, x0 0.00 : ffff800010121f08: ldr x23, [x0, #2464] : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff800010121f0c: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.00 : ffff800010121f10: ldr x22, [x23, #4384] : find_energy_efficient_cpu(): : if (!pd || READ_ONCE(rd->overutilized)) 0.00 : ffff800010121f14: cbz x22, ffff800010121f94 : __read_once_size(): 0.00 : ffff800010121f18: ldr w0, [x23, #92] : find_energy_efficient_cpu(): 0.00 : ffff800010121f1c: cbnz w0, ffff800010121f94 : sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 0.00 : ffff800010121f20: adrp x0, ffff80001151f000 0.00 : ffff800010121f24: add x0, x0, #0x140 : __my_cpu_offset(): 0.00 : ffff800010121f28: mrs x1, tpidr_el1 : __read_once_size(): 0.00 : ffff800010121f2c: ldr x0, [x0, x1] : find_energy_efficient_cpu(): : while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 0.00 : ffff800010121f30: cbz x0, ffff800010121f94 : test_bit(): 0.00 : ffff800010121f34: add w1, w21, #0x3f 0.00 : ffff800010121f38: cmp w21, #0x0 0.00 : ffff800010121f3c: csel w1, w1, w21, lt // lt = tstop : cpumask_test_cpu(): 0.00 : ffff800010121f40: add x25, x0, #0x88 0.00 : ffff800010121f44: stp x26, x27, [x29, #72] : test_bit(): 0.00 : ffff800010121f48: and w4, w21, #0x3f 0.00 : ffff800010121f4c: asr w1, w1, #6 0.00 : ffff800010121f50: str x28, [x29, #88] 0.00 : ffff800010121f54: sxtw x1, w1 0.00 : ffff800010121f58: lsl x3, x1, #3 0.00 : ffff800010121f5c: ldr x1, [x25, x1, lsl #3] 0.00 : ffff800010121f60: lsr x1, x1, x4 : find_energy_efficient_cpu(): 0.00 : ffff800010121f64: tbnz w1, #0, ffff800010122274 0.00 : ffff800010121f68: mov x2, #0x1 // #1 0.00 : ffff800010121f6c: lsl x2, x2, x4 0.00 : ffff800010121f70: b ffff800010121f80 : test_bit(): 0.00 : ffff800010121f74: ldr x1, [x25, x3] : find_energy_efficient_cpu(): 0.00 : ffff800010121f78: tst x1, x2 0.00 : ffff800010121f7c: b.ne ffff800010122274 // b.any : sd = sd->parent; 0.00 : ffff800010121f80: ldr x0, [x0] : cpumask_test_cpu(): 0.00 : ffff800010121f84: add x25, x0, #0x88 : find_energy_efficient_cpu(): : while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 0.00 : ffff800010121f88: cbnz x0, ffff800010121f74 0.00 : ffff800010121f8c: ldp x26, x27, [x29, #72] 0.00 : ffff800010121f90: ldr x28, [x29, #88] : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff800010121f94: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff800010121f98: b ffff800010121dbc : wake_wide(): : int factor = this_cpu_read(sd_llc_size); 0.00 : ffff800010121f9c: bl ffff800010cd2e78 : if (master < slave) 0.00 : ffff800010121fa0: cmp w22, w23 0.00 : ffff800010121fa4: csel w0, w23, w22, cc // cc = lo, ul, last 0.00 : ffff800010121fa8: csel w22, w22, w23, cc // cc = lo, ul, last : if (slave < factor || master < slave * factor) 0.00 : ffff800010121fac: cmp w25, w22 0.00 : ffff800010121fb0: b.ls ffff800010121e10 // b.plast : arch_static_branch(): 0.00 : ffff800010121fb4: nop 0.00 : ffff800010121fb8: sxtw x25, w24 : test_bit(): 0.00 : ffff800010121fbc: cmp w24, #0x0 0.00 : ffff800010121fc0: add w0, w24, #0x3f 0.00 : ffff800010121fc4: csel w0, w0, w24, lt // lt = tstop 0.05 : ffff800010121fc8: ldr x1, [x20, #736] 0.00 : ffff800010121fcc: asr w0, w0, #6 0.00 : ffff800010121fd0: sxtw x0, w0 0.62 : ffff800010121fd4: ldr x23, [x1, x0, lsl #3] : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff800010121fd8: bl ffff80001015c1b0 <__rcu_read_lock> : select_task_rq_fair(): : for_each_domain(cpu, tmp) { 0.54 : ffff800010121fdc: ldr x0, [x29, #200] 0.00 : ffff800010121fe0: add x1, x0, #0x928 0.56 : ffff800010121fe4: ldr x0, [x29, #184] : test_bit(): 0.00 : ffff800010121fe8: lsr x23, x23, x24 : select_task_rq_fair(): : int want_affine = 0; 0.00 : ffff800010121fec: and w23, w23, #0x1 : for_each_domain(cpu, tmp) { 0.00 : ffff800010121ff0: add x0, x0, #0xe80 0.15 : ffff800010121ff4: ldr x1, [x1, x25, lsl #3] 0.00 : ffff800010121ff8: add x0, x0, x1 : __read_once_size(): 1.14 : ffff800010121ffc: ldr x22, [x0, #2472] : select_task_rq_fair(): 0.11 : ffff800010122000: cbnz x22, ffff800010121e48 0.00 : ffff800010122004: nop : new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 0.00 : ffff800010122008: mov w2, w21 0.00 : ffff80001012200c: mov w1, w21 0.00 : ffff800010122010: mov x0, x20 0.00 : ffff800010122014: bl ffff80001011e278 0.00 : ffff800010122018: mov w21, w0 : if (want_affine) 0.00 : ffff80001012201c: cbz w23, ffff800010121eb8 : get_current(): 0.24 : ffff800010122020: mrs x0, sp_el0 : select_task_rq_fair(): : current->recent_used_cpu = cpu; 0.16 : ffff800010122024: str w24, [x0, #96] 0.00 : ffff800010122028: b ffff800010121eb8 : if (tmp->flags & sd_flag) 0.00 : ffff80001012202c: tst w19, w0 0.00 : ffff800010122030: b.eq ffff800010121ea8 // b.none 0.00 : ffff800010122034: mov x27, x22 0.00 : ffff800010122038: b ffff800010121e98 : capacity_orig_of(): : #endif : : #ifdef CONFIG_SMP : static inline unsigned long capacity_orig_of(int cpu) : { : return cpu_rq(cpu)->cpu_capacity_orig; 0.00 : ffff80001012203c: ldr x0, [x29, #200] 0.00 : ffff800010122040: sxtw x25, w24 0.00 : ffff800010122044: add x2, x0, #0x928 0.00 : ffff800010122048: ldr x0, [x29, #184] 0.00 : ffff80001012204c: add x0, x0, #0xe80 0.00 : ffff800010122050: ldr x1, [x2, x25, lsl #3] 0.00 : ffff800010122054: ldr x2, [x2, w21, sxtw #3] 0.00 : ffff800010122058: add x1, x0, x1 0.00 : ffff80001012205c: add x0, x2, x0 : wake_cap(): : max_cap = cpu_rq(cpu)->rd->max_cpu_capacity; 0.00 : ffff800010122060: ldr x2, [x1, #2464] : min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu)); 0.00 : ffff800010122064: ldr x0, [x0, #2488] 0.00 : ffff800010122068: ldr x22, [x1, #2488] : max_cap = cpu_rq(cpu)->rd->max_cpu_capacity; 0.00 : ffff80001012206c: ldr x1, [x2, #4376] : min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu)); 0.00 : ffff800010122070: cmp x0, x22 0.00 : ffff800010122074: csel x22, x0, x22, ls // ls = plast : if (max_cap - min_cap < max_cap >> 3) 0.00 : ffff800010122078: sub x0, x1, x22 0.00 : ffff80001012207c: cmp x0, x1, asr #3 0.00 : ffff800010122080: b.lt ffff800010121fbc // b.tstop : sync_entity_load_avg(): : last_update_time = cfs_rq_last_update_time(cfs_rq); 0.00 : ffff800010122084: ldr x0, [x20, #320] : __update_load_avg_blocked_se(last_update_time, se); 0.00 : ffff800010122088: add x1, x20, #0xc0 0.00 : ffff80001012208c: ldr x0, [x0, #128] 0.00 : ffff800010122090: bl ffff800010134738 <__update_load_avg_blocked_se> : __read_once_size(): 0.00 : ffff800010122094: ldr x3, [x20, #432] : task_fits_capacity(): : return fits_capacity(uclamp_task_util(p), capacity); 0.00 : ffff800010122098: lsl x0, x22, #10 : __read_once_size(): 0.00 : ffff80001012209c: ldr x2, [x20, #440] : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff8000101220a0: lsr x1, x2, #32 0.00 : ffff8000101220a4: cmp w1, w2 0.00 : ffff8000101220a8: csel w1, w1, w2, hi // hi = pmore 0.00 : ffff8000101220ac: orr w1, w1, #0x1 : task_util_est(): : return max(task_util(p), _task_util_est(p)); 0.00 : ffff8000101220b0: cmp x1, x3 0.00 : ffff8000101220b4: csel x1, x1, x3, cs // cs = hs, nlast : task_fits_capacity(): : return fits_capacity(uclamp_task_util(p), capacity); 0.00 : ffff8000101220b8: add x1, x1, x1, lsl #2 : select_task_rq_fair(): : want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu) && 0.00 : ffff8000101220bc: cmp x0, x1, lsl #8 0.00 : ffff8000101220c0: b.ls ffff800010121e20 // b.plast 0.00 : ffff8000101220c4: b ffff800010121fbc 0.00 : ffff8000101220c8: adrp x0, ffff800011528000 0.00 : ffff8000101220cc: sxtw x25, w24 0.00 : ffff8000101220d0: str x0, [x29, #184] 0.00 : ffff8000101220d4: adrp x0, ffff800011909000 0.00 : ffff8000101220d8: str x0, [x29, #200] 0.00 : ffff8000101220dc: b ffff800010121e20 : bitmap_intersects(): : const unsigned long *src2, unsigned int nbits) : { : if (small_const_nbits(nbits)) : return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; : else : return __bitmap_intersects(src1, src2, nbits); 0.00 : ffff8000101220e0: ldr x1, [x20, #736] 0.00 : ffff8000101220e4: mov w2, #0x100 // #256 0.00 : ffff8000101220e8: add x0, x27, #0x88 0.00 : ffff8000101220ec: bl ffff800010485268 <__bitmap_intersects> : find_idlest_cpu(): : if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 0.00 : ffff8000101220f0: cbz w0, ffff800010122534 0.00 : ffff8000101220f4: str x26, [x29, #72] 0.00 : ffff8000101220f8: str x28, [x29, #88] : if (!(sd_flag & SD_BALANCE_FORK)) 0.00 : ffff8000101220fc: tbz w19, #3, ffff800010122520 : for_each_domain(cpu, tmp) { 0.00 : ffff800010122100: ldr x0, [x29, #184] 0.00 : ffff800010122104: adrp x25, ffff80001190a000 <__per_cpu_offset+0x6d8> : select_task_rq_fair(): : int cpu = smp_processor_id(); 0.00 : ffff800010122108: mov w21, w24 : find_idlest_group_cpu(): : for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 0.00 : ffff80001012210c: add x25, x25, #0x2f0 : find_idlest_cpu(): : for_each_domain(cpu, tmp) { 0.00 : ffff800010122110: add x0, x0, #0xe80 0.00 : ffff800010122114: str x0, [x29, #192] : if (!(sd->flags & sd_flag)) { 0.00 : ffff800010122118: ldr w0, [x27, #56] 0.00 : ffff80001012211c: tst w19, w0 0.00 : ffff800010122120: b.eq ffff800010122220 // b.none : group = find_idlest_group(sd, p, cpu, sd_flag); 0.00 : ffff800010122124: mov w2, w21 0.00 : ffff800010122128: mov x1, x20 0.00 : ffff80001012212c: mov x0, x27 0.00 : ffff800010122130: bl ffff80001011d3a8 : if (!group) { 0.00 : ffff800010122134: cbz x0, ffff800010122220 : find_idlest_group_cpu(): : if (group->group_weight == 1) 0.00 : ffff800010122138: ldr w1, [x0, #12] 0.00 : ffff80001012213c: add x0, x0, #0x20 0.00 : ffff800010122140: str x0, [x29, #184] 0.00 : ffff800010122144: cmp w1, #0x1 0.00 : ffff800010122148: b.eq ffff800010122558 // b.none : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff80001012214c: ldr x0, [x29, #200] : find_idlest_group_cpu(): : for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 0.00 : ffff800010122150: mov w23, #0xffffffff // #-1 : int shallowest_idle_cpu = -1; 0.00 : ffff800010122154: mov w24, w23 : unsigned int min_exit_latency = UINT_MAX; 0.00 : ffff800010122158: str w23, [x29, #168] : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff80001012215c: add x22, x0, #0x928 : find_idlest_group_cpu(): : if (group->group_weight == 1) 0.00 : ffff800010122160: str w21, [x29, #180] : unsigned long load, min_load = ULONG_MAX; 0.00 : ffff800010122164: mov x0, #0xffffffffffffffff // #-1 : u64 latest_idle_timestamp = 0; 0.00 : ffff800010122168: stp x0, xzr, [x29, #152] 0.00 : ffff80001012216c: nop : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff800010122170: ldp x1, x26, [x29, #184] : find_idlest_group_cpu(): : for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 0.00 : ffff800010122174: mov w0, w23 0.00 : ffff800010122178: ldr x2, [x20, #736] 0.00 : ffff80001012217c: bl ffff800010cb93f0 0.00 : ffff800010122180: sxtw x28, w0 0.00 : ffff800010122184: ldr w1, [x25] 0.00 : ffff800010122188: mov x23, x28 0.00 : ffff80001012218c: cmp w28, w1 0.00 : ffff800010122190: b.cs ffff80001012244c // b.hs, b.nlast : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff800010122194: ldr x1, [x22, x28, lsl #3] 0.00 : ffff800010122198: add x1, x26, x1 0.00 : ffff80001012219c: ldr w2, [x1, #4] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff8000101221a0: ldr w1, [x1, #160] 0.00 : ffff8000101221a4: cmp w2, #0x0 0.00 : ffff8000101221a8: ccmp w1, w2, #0x0, ne // ne = any 0.00 : ffff8000101221ac: b.eq ffff8000101226b0 // b.none : find_idlest_group_cpu(): : if (available_idle_cpu(i)) { 0.00 : ffff8000101221b0: bl ffff800010118870 0.00 : ffff8000101221b4: cbz w0, ffff8000101221ec : struct rq *rq = cpu_rq(i); 0.00 : ffff8000101221b8: ldr x0, [x22, x28, lsl #3] 0.00 : ffff8000101221bc: add x4, x26, x0 : struct cpuidle_state *idle = idle_get_state(rq); 0.00 : ffff8000101221c0: ldr x0, [x4, #2984] : if (idle && idle->exit_latency < min_exit_latency) { 0.00 : ffff8000101221c4: cbz x0, ffff800010122238 0.00 : ffff8000101221c8: ldr w0, [x0, #68] 0.00 : ffff8000101221cc: ldr w1, [x29, #168] 0.00 : ffff8000101221d0: cmp w0, w1 0.00 : ffff8000101221d4: b.cs ffff800010122234 // b.hs, b.nlast : latest_idle_timestamp = rq->idle_stamp; 0.00 : ffff8000101221d8: ldr x1, [x4, #2816] : shallowest_idle_cpu = i; 0.00 : ffff8000101221dc: mov w24, w23 : latest_idle_timestamp = rq->idle_stamp; 0.00 : ffff8000101221e0: str x1, [x29, #160] 0.00 : ffff8000101221e4: str w0, [x29, #168] 0.00 : ffff8000101221e8: b ffff800010122170 : } else if (shallowest_idle_cpu == -1) { 0.00 : ffff8000101221ec: cmn w24, #0x1 0.00 : ffff8000101221f0: b.ne ffff800010122170 // b.any : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff8000101221f4: ldr x0, [x22, x28, lsl #3] : find_idlest_group_cpu(): : if (load < min_load) { 0.00 : ffff8000101221f8: ldr x1, [x29, #152] : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff8000101221fc: add x4, x26, x0 : find_idlest_group_cpu(): : if (load < min_load) { 0.00 : ffff800010122200: ldr w2, [x29, #180] : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff800010122204: ldr x0, [x4, #288] : find_idlest_group_cpu(): : if (load < min_load) { 0.00 : ffff800010122208: cmp x0, x1 0.00 : ffff80001012220c: csel w28, w23, w2, cc // cc = lo, ul, last 0.00 : ffff800010122210: csel x0, x0, x1, cc // cc = lo, ul, last 0.00 : ffff800010122214: str x0, [x29, #152] 0.00 : ffff800010122218: str w28, [x29, #180] 0.00 : ffff80001012221c: b ffff800010122170 : find_idlest_cpu(): : sd = sd->child; 0.00 : ffff800010122220: ldr x27, [x27, #8] : while (sd) { 0.00 : ffff800010122224: cbnz x27, ffff800010122118 : if (weight <= tmp->span_weight) 0.00 : ffff800010122228: ldp x26, x27, [x29, #72] 0.00 : ffff80001012222c: ldr x28, [x29, #88] 0.00 : ffff800010122230: b ffff800010121eb8 : find_idlest_group_cpu(): : } else if ((!idle || idle->exit_latency == min_exit_latency) && 0.00 : ffff800010122234: b.ne ffff800010122170 // b.any 0.00 : ffff800010122238: ldr x1, [x29, #160] : rq->idle_stamp > latest_idle_timestamp) { 0.00 : ffff80001012223c: ldr x0, [x4, #2816] : } else if ((!idle || idle->exit_latency == min_exit_latency) && 0.00 : ffff800010122240: cmp x0, x1 0.00 : ffff800010122244: csel x0, x0, x1, hi // hi = pmore 0.00 : ffff800010122248: csel w24, w23, w24, hi // hi = pmore 0.00 : ffff80001012224c: str x0, [x29, #160] 0.00 : ffff800010122250: b ffff800010122170 : select_task_rq_fair(): : if (cpu != prev_cpu) 3.15 : ffff800010122254: cmp w24, w21 0.00 : ffff800010122258: b.ne ffff8000101224cc // b.any : return new_cpu; 0.00 : ffff80001012225c: mov w2, w21 : } else if (sd_flag & SD_BALANCE_WAKE) { /* XXX always ? */ 6.28 : ffff800010122260: ldr w0, [x29, #180] 0.00 : ffff800010122264: cbnz w0, ffff800010122540 0.00 : ffff800010122268: mov w21, w2 0.00 : ffff80001012226c: ldr x27, [x29, #80] 0.00 : ffff800010122270: b ffff800010121eb8 : find_energy_efficient_cpu(): : unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 0.00 : ffff800010122274: mov x2, #0xffffffffffffffff // #-1 0.00 : ffff800010122278: str x2, [x29, #104] 0.00 : ffff80001012227c: str x2, [x29, #128] : sync_entity_load_avg(): : __update_load_avg_blocked_se(last_update_time, se); 0.00 : ffff800010122280: add x1, x20, #0xc0 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010122284: ldr x2, [x29, #184] : sync_entity_load_avg(): : last_update_time = cfs_rq_last_update_time(cfs_rq); 0.00 : ffff800010122288: ldr x0, [x20, #320] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff80001012228c: add x2, x2, #0xe80 0.00 : ffff800010122290: str x2, [x29, #160] 0.00 : ffff800010122294: ldr x2, [x29, #200] : sync_entity_load_avg(): : __update_load_avg_blocked_se(last_update_time, se); 0.00 : ffff800010122298: ldr x0, [x0, #128] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff80001012229c: add x2, x2, #0x928 : find_energy_efficient_cpu(): : unsigned long cpu_cap, util, base_energy = 0; 0.00 : ffff8000101222a0: str xzr, [x29, #144] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff8000101222a4: str x2, [x29, #152] : __read_once_size(): 0.00 : ffff8000101222a8: str w21, [x29, #192] : sync_entity_load_avg(): : __update_load_avg_blocked_se(last_update_time, se); 0.00 : ffff8000101222ac: bl ffff800010134738 <__update_load_avg_blocked_se> : __read_once_size(): 0.00 : ffff8000101222b0: ldr x0, [x20, #432] 0.00 : ffff8000101222b4: ldr x0, [x20, #440] : find_energy_efficient_cpu(): : base_energy_pd = compute_energy(p, -1, pd); 0.00 : ffff8000101222b8: mov w1, #0xffffffff // #-1 0.00 : ffff8000101222bc: mov x2, x22 0.00 : ffff8000101222c0: mov x0, x20 0.00 : ffff8000101222c4: bl ffff80001011d1f8 : base_energy += base_energy_pd; 0.00 : ffff8000101222c8: ldr x1, [x29, #144] : int max_spare_cap_cpu = -1; 0.00 : ffff8000101222cc: mov w23, #0xffffffff // #-1 : base_energy_pd = compute_energy(p, -1, pd); 0.00 : ffff8000101222d0: str x0, [x29, #136] : for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { 0.00 : ffff8000101222d4: mov w28, w23 : base_energy += base_energy_pd; 0.00 : ffff8000101222d8: add x0, x1, x0 0.00 : ffff8000101222dc: str x0, [x29, #144] : for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { 0.00 : ffff8000101222e0: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101222e4: add x27, x0, #0x2f0 : unsigned long cur_delta, spare_cap, max_spare_cap = 0; 0.00 : ffff8000101222e8: mov x26, #0x0 // #0 0.00 : ffff8000101222ec: nop : for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { 0.00 : ffff8000101222f0: ldr x1, [x22] 0.00 : ffff8000101222f4: mov x2, x25 0.00 : ffff8000101222f8: mov w0, w28 0.00 : ffff8000101222fc: add x1, x1, #0x10 0.00 : ffff800010122300: bl ffff800010cb93f0 0.00 : ffff800010122304: ldr w5, [x27] 0.00 : ffff800010122308: mov w28, w0 : test_bit(): 0.00 : ffff80001012230c: add w4, w0, #0x3f : find_energy_efficient_cpu(): : util = cpu_util_next(cpu, p, cpu); 0.00 : ffff800010122310: mov w2, w0 0.00 : ffff800010122314: mov x1, x20 : for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { 0.00 : ffff800010122318: cmp w0, w5 0.00 : ffff80001012231c: b.cs ffff800010122384 // b.hs, b.nlast : test_bit(): 0.00 : ffff800010122320: cmp w28, #0x0 0.00 : ffff800010122324: ldr x5, [x20, #736] 0.00 : ffff800010122328: csel w4, w4, w28, lt // lt = tstop 0.00 : ffff80001012232c: asr w4, w4, #6 0.00 : ffff800010122330: sxtw x4, w4 0.00 : ffff800010122334: ldr x4, [x5, x4, lsl #3] 0.00 : ffff800010122338: lsr x4, x4, x28 : find_energy_efficient_cpu(): : if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 0.00 : ffff80001012233c: tbz w4, #0, ffff8000101222f0 : util = cpu_util_next(cpu, p, cpu); 0.00 : ffff800010122340: bl ffff80001011c760 0.00 : ffff800010122344: mov x5, x0 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010122348: ldp x0, x1, [x29, #152] 0.00 : ffff80001012234c: ldr x0, [x0, w28, sxtw #3] 0.00 : ffff800010122350: add x0, x0, x1 : find_energy_efficient_cpu(): : if (!fits_capacity(util, cpu_cap)) 0.00 : ffff800010122354: add x1, x5, x5, lsl #2 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010122358: ldr x4, [x0, #2480] : find_energy_efficient_cpu(): : if (!fits_capacity(util, cpu_cap)) 0.00 : ffff80001012235c: lsl x0, x4, #10 0.00 : ffff800010122360: cmp x0, x1, lsl #8 0.00 : ffff800010122364: b.ls ffff8000101222f0 // b.plast : if (cpu == prev_cpu) { 0.00 : ffff800010122368: cmp w21, w28 0.00 : ffff80001012236c: b.eq ffff8000101223dc // b.none : spare_cap = cpu_cap - util; 0.00 : ffff800010122370: sub x0, x4, x5 : if (spare_cap > max_spare_cap) { 0.00 : ffff800010122374: cmp x0, x26 0.00 : ffff800010122378: csel w23, w28, w23, hi // hi = pmore 0.00 : ffff80001012237c: csel x26, x0, x26, hi // hi = pmore 0.00 : ffff800010122380: b ffff8000101222f0 : if (max_spare_cap_cpu >= 0 && max_spare_cap_cpu != prev_cpu) { 0.00 : ffff800010122384: cmp w23, #0x0 0.00 : ffff800010122388: ccmp w21, w23, #0x4, ge // ge = tcont 0.00 : ffff80001012238c: b.ne ffff800010122414 // b.any : for (; pd; pd = pd->next) { 0.00 : ffff800010122390: ldr x22, [x22, #8] 0.00 : ffff800010122394: cbnz x22, ffff8000101222b8 : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff800010122398: bl ffff800010160d20 <__rcu_read_unlock> : find_energy_efficient_cpu(): : if (prev_delta == ULONG_MAX) 0.00 : ffff80001012239c: ldr x0, [x29, #104] 0.00 : ffff8000101223a0: cmn x0, #0x1 0.00 : ffff8000101223a4: b.eq ffff8000101223c8 // b.none : if ((prev_delta - best_delta) > ((prev_delta + base_energy) >> 4)) 0.00 : ffff8000101223a8: ldr x1, [x29, #144] 0.00 : ffff8000101223ac: ldr x2, [x29, #128] 0.00 : ffff8000101223b0: add x1, x1, x0 0.00 : ffff8000101223b4: sub x0, x0, x2 0.00 : ffff8000101223b8: cmp x0, x1, lsr #4 0.00 : ffff8000101223bc: ldr w0, [x29, #192] 0.00 : ffff8000101223c0: csel w0, w0, w21, hi // hi = pmore 0.00 : ffff8000101223c4: str w0, [x29, #192] : select_task_rq_fair(): : if (new_cpu >= 0) 0.00 : ffff8000101223c8: ldr w0, [x29, #192] 0.00 : ffff8000101223cc: ldp x26, x27, [x29, #72] 0.00 : ffff8000101223d0: ldr x28, [x29, #88] 0.00 : ffff8000101223d4: tbz w0, #31, ffff800010121ec0 0.00 : ffff8000101223d8: b ffff800010121dbc : find_energy_efficient_cpu(): : prev_delta = compute_energy(p, prev_cpu, pd); 0.00 : ffff8000101223dc: mov w1, w21 0.00 : ffff8000101223e0: stp x4, x5, [x29, #112] 0.00 : ffff8000101223e4: mov x2, x22 0.00 : ffff8000101223e8: mov x0, x20 0.00 : ffff8000101223ec: bl ffff80001011d1f8 : prev_delta -= base_energy_pd; 0.00 : ffff8000101223f0: ldr x1, [x29, #136] : best_delta = min(best_delta, prev_delta); 0.00 : ffff8000101223f4: ldp x4, x5, [x29, #112] : prev_delta -= base_energy_pd; 0.00 : ffff8000101223f8: sub x0, x0, x1 : best_delta = min(best_delta, prev_delta); 0.00 : ffff8000101223fc: ldr x1, [x29, #128] : prev_delta -= base_energy_pd; 0.00 : ffff800010122400: str x0, [x29, #104] : best_delta = min(best_delta, prev_delta); 0.00 : ffff800010122404: cmp x1, x0 0.00 : ffff800010122408: csel x0, x1, x0, ls // ls = plast 0.00 : ffff80001012240c: str x0, [x29, #128] 0.00 : ffff800010122410: b ffff800010122370 : cur_delta = compute_energy(p, max_spare_cap_cpu, pd); 0.00 : ffff800010122414: mov x2, x22 0.00 : ffff800010122418: mov w1, w23 0.00 : ffff80001012241c: mov x0, x20 0.00 : ffff800010122420: bl ffff80001011d1f8 : cur_delta -= base_energy_pd; 0.00 : ffff800010122424: ldr x1, [x29, #136] : if (cur_delta < best_delta) { 0.00 : ffff800010122428: ldr w2, [x29, #192] : cur_delta -= base_energy_pd; 0.00 : ffff80001012242c: sub x0, x0, x1 : if (cur_delta < best_delta) { 0.00 : ffff800010122430: ldr x1, [x29, #128] 0.00 : ffff800010122434: cmp x1, x0 0.00 : ffff800010122438: csel w28, w23, w2, hi // hi = pmore 0.00 : ffff80001012243c: csel x0, x0, x1, hi // hi = pmore 0.00 : ffff800010122440: str x0, [x29, #128] 0.00 : ffff800010122444: str w28, [x29, #192] 0.00 : ffff800010122448: b ffff800010122390 : find_idlest_group_cpu(): : return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 0.00 : ffff80001012244c: ldr w0, [x29, #180] 0.00 : ffff800010122450: cmn w24, #0x1 0.00 : ffff800010122454: csel w0, w0, w24, eq // eq = none 0.00 : ffff800010122458: str w0, [x29, #180] : find_idlest_cpu(): : if (new_cpu == cpu) { 0.00 : ffff80001012245c: ldr w0, [x29, #180] 0.00 : ffff800010122460: cmp w0, w21 0.00 : ffff800010122464: b.eq ffff800010122220 // b.none : for_each_domain(cpu, tmp) { 0.00 : ffff800010122468: ldr x0, [x29, #200] 0.00 : ffff80001012246c: ldr w3, [x29, #180] 0.00 : ffff800010122470: add x1, x0, #0x928 0.00 : ffff800010122474: ldr x0, [x29, #192] : weight = sd->span_weight; 0.00 : ffff800010122478: ldr w2, [x27, #128] : for_each_domain(cpu, tmp) { 0.00 : ffff80001012247c: ldr x1, [x1, w3, sxtw #3] 0.00 : ffff800010122480: add x0, x0, x1 : __read_once_size(): 0.00 : ffff800010122484: ldr x0, [x0, #2472] : find_idlest_cpu(): 0.00 : ffff800010122488: cbz x0, ffff8000101226b8 : if (weight <= tmp->span_weight) 0.00 : ffff80001012248c: ldr w1, [x0, #128] 0.00 : ffff800010122490: cmp w1, w2 0.00 : ffff800010122494: b.cs ffff8000101226b8 // b.hs, b.nlast : sd = NULL; 0.00 : ffff800010122498: mov x27, #0x0 // #0 0.00 : ffff80001012249c: b ffff8000101224ac : if (weight <= tmp->span_weight) 0.00 : ffff8000101224a0: ldr w1, [x0, #128] 0.00 : ffff8000101224a4: cmp w2, w1 0.00 : ffff8000101224a8: b.ls ffff8000101224c0 // b.plast : if (tmp->flags & sd_flag) 0.00 : ffff8000101224ac: ldr w1, [x0, #56] 0.00 : ffff8000101224b0: tst w19, w1 0.00 : ffff8000101224b4: csel x27, x27, x0, eq // eq = none : for_each_domain(cpu, tmp) { 0.00 : ffff8000101224b8: ldr x0, [x0] 0.00 : ffff8000101224bc: cbnz x0, ffff8000101224a0 : if (weight <= tmp->span_weight) 0.00 : ffff8000101224c0: ldr w21, [x29, #180] : while (sd) { 0.00 : ffff8000101224c4: cbnz x27, ffff800010122118 0.00 : ffff8000101224c8: b ffff800010122228 : wake_affine_idle(): : if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 0.40 : ffff8000101224cc: mov w0, w24 0.00 : ffff8000101224d0: bl ffff800010118870 0.19 : ffff8000101224d4: cbnz w0, ffff80001012256c 0.01 : ffff8000101224d8: ldr x0, [x29, #200] : if (sync && cpu_rq(this_cpu)->nr_running == 1) 0.31 : ffff8000101224dc: ldr w1, [x29, #168] 0.00 : ffff8000101224e0: add x0, x0, #0x928 0.01 : ffff8000101224e4: ldr x0, [x0, x25, lsl #3] 0.00 : ffff8000101224e8: cbz w1, ffff800010122694 0.00 : ffff8000101224ec: ldr x1, [x29, #184] 0.00 : ffff8000101224f0: add x1, x1, #0xe80 0.00 : ffff8000101224f4: add x0, x0, x1 0.00 : ffff8000101224f8: ldr w1, [x0, #4] 0.00 : ffff8000101224fc: cmp w1, #0x1 0.00 : ffff800010122500: b.ne ffff8000101226a8 // b.any : wake_affine(): : if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 0.00 : ffff800010122504: cmp w24, #0x100 0.00 : ffff800010122508: b.eq ffff8000101226c0 // b.none : select_task_rq_fair(): : } else if (sd_flag & SD_BALANCE_WAKE) { /* XXX always ? */ 0.00 : ffff80001012250c: ldr w0, [x29, #180] 0.00 : ffff800010122510: cbnz w0, ffff80001012253c 0.00 : ffff800010122514: mov w21, w24 0.00 : ffff800010122518: ldr x27, [x29, #80] 0.00 : ffff80001012251c: b ffff800010121eb8 : sync_entity_load_avg(): : last_update_time = cfs_rq_last_update_time(cfs_rq); 0.00 : ffff800010122520: ldr x0, [x20, #320] : __update_load_avg_blocked_se(last_update_time, se); 0.00 : ffff800010122524: add x1, x20, #0xc0 0.00 : ffff800010122528: ldr x0, [x0, #128] 0.00 : ffff80001012252c: bl ffff800010134738 <__update_load_avg_blocked_se> 0.00 : ffff800010122530: b ffff800010122100 0.00 : ffff800010122534: ldr x27, [x29, #80] 0.00 : ffff800010122538: b ffff800010121eb8 : select_task_rq_fair(): : new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 0.00 : ffff80001012253c: mov w2, w24 0.00 : ffff800010122540: mov w1, w21 0.00 : ffff800010122544: mov x0, x20 0.00 : ffff800010122548: bl ffff80001011e278 0.15 : ffff80001012254c: ldr x27, [x29, #80] 0.00 : ffff800010122550: mov w21, w0 0.00 : ffff800010122554: b ffff800010122020 : cpumask_first(): : return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits); 0.00 : ffff800010122558: mov x2, #0x0 // #0 0.00 : ffff80001012255c: mov x1, #0x100 // #256 0.00 : ffff800010122560: bl ffff80001048daa8 : find_idlest_group_cpu(): : return cpumask_first(sched_group_span(group)); 0.00 : ffff800010122564: str w0, [x29, #180] 0.00 : ffff800010122568: b ffff80001012245c : wake_affine_idle(): : if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 0.00 : ffff80001012256c: mov w1, w21 0.00 : ffff800010122570: mov w0, w24 0.00 : ffff800010122574: bl ffff800010117770 0.00 : ffff800010122578: tst w0, #0xff 0.00 : ffff80001012257c: b.eq ffff8000101224d8 // b.none : return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 0.00 : ffff800010122580: mov w0, w21 0.00 : ffff800010122584: bl ffff800010118870 0.00 : ffff800010122588: cmp w0, #0x0 0.00 : ffff80001012258c: csel w2, w21, w24, ne // ne = any : wake_affine(): : if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 0.00 : ffff800010122590: cmp w2, #0x100 0.00 : ffff800010122594: b.ne ffff800010122260 // b.any : wake_affine_weight(): : this_eff_load = cpu_load(cpu_rq(this_cpu)); 0.00 : ffff800010122598: ldr x0, [x29, #200] 0.00 : ffff80001012259c: add x1, x0, #0x928 0.00 : ffff8000101225a0: ldr x0, [x29, #184] 0.00 : ffff8000101225a4: add x0, x0, #0xe80 : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff8000101225a8: ldr x1, [x1, x25, lsl #3] 0.00 : ffff8000101225ac: add x0, x0, x1 0.00 : ffff8000101225b0: ldr x7, [x0, #288] : wake_affine_weight(): : if (sync) { 0.00 : ffff8000101225b4: ldr w0, [x29, #168] : this_eff_load = cpu_load(cpu_rq(this_cpu)); 0.00 : ffff8000101225b8: mov x6, x7 : if (sync) { 0.00 : ffff8000101225bc: cbz w0, ffff8000101225f4 : get_current(): 0.00 : ffff8000101225c0: mrs x8, sp_el0 : task_cfs_rq(): : return p->se.cfs_rq; 0.00 : ffff8000101225c4: ldr x6, [x8, #320] : task_h_load(): : : static unsigned long task_h_load(struct task_struct *p) : { : struct cfs_rq *cfs_rq = task_cfs_rq(p); : : update_cfs_rq_h_load(cfs_rq); 0.00 : ffff8000101225c8: mov x0, x6 0.00 : ffff8000101225cc: bl ffff80001011c898 : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.00 : ffff8000101225d0: ldr x2, [x6, #280] 0.00 : ffff8000101225d4: ldr x0, [x8, #416] 0.00 : ffff8000101225d8: ldr x1, [x6, #160] 0.00 : ffff8000101225dc: add x1, x1, #0x1 0.00 : ffff8000101225e0: mul x0, x0, x2 : div64_u64(): : * : * Return: dividend / divisor : */ : static inline u64 div64_u64(u64 dividend, u64 divisor) : { : return dividend / divisor; 0.00 : ffff8000101225e4: udiv x0, x0, x1 : wake_affine_weight(): : if (current_load > this_eff_load) 0.00 : ffff8000101225e8: cmp x7, x0 0.00 : ffff8000101225ec: b.cc ffff800010122684 // b.lo, b.ul, b.last : this_eff_load -= current_load; 0.00 : ffff8000101225f0: sub x6, x7, x0 : task_cfs_rq(): : return p->se.cfs_rq; 1.64 : ffff8000101225f4: ldr x7, [x20, #320] : task_h_load(): : update_cfs_rq_h_load(cfs_rq); 0.00 : ffff8000101225f8: mov x0, x7 0.00 : ffff8000101225fc: bl ffff80001011c898 : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.32 : ffff800010122600: ldr x3, [x7, #280] 0.53 : ffff800010122604: ldr x1, [x20, #416] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 9.04 : ffff800010122608: ldr x0, [x29, #200] : task_h_load(): : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.04 : ffff80001012260c: ldr x4, [x7, #160] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010122610: add x5, x0, #0x928 26.79 : ffff800010122614: ldr x0, [x29, #184] : task_h_load(): : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.00 : ffff800010122618: mul x1, x1, x3 0.00 : ffff80001012261c: add x4, x4, #0x1 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010122620: add x2, x0, #0xe80 : wake_affine_weight(): : prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 0.01 : ffff800010122624: ldr w0, [x22, #44] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.06 : ffff800010122628: ldr x3, [x5, w21, sxtw #3] : div64_u64(): 0.00 : ffff80001012262c: udiv x1, x1, x4 : capacity_of(): 0.00 : ffff800010122630: add x3, x2, x3 : wake_affine_weight(): : prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 0.00 : ffff800010122634: sub w0, w0, #0x64 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.03 : ffff800010122638: ldr x5, [x5, x25, lsl #3] : wake_affine_weight(): : prev_eff_load += 1; 0.45 : ffff80001012263c: ldr x7, [x29, #168] : prev_eff_load -= task_load; 0.00 : ffff800010122640: ldr x4, [x3, #288] : prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 0.00 : ffff800010122644: lsr w0, w0, #1 : this_eff_load += task_load; 0.00 : ffff800010122648: add x6, x6, x1 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff80001012264c: add x5, x5, x2 : wake_affine_weight(): : prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 6.39 : ffff800010122650: add w0, w0, #0x64 : prev_eff_load -= task_load; 0.00 : ffff800010122654: sub x2, x4, x1 : this_eff_load *= 100; 0.00 : ffff800010122658: add x1, x6, x6, lsl #1 : this_eff_load *= capacity_of(prev_cpu); 0.00 : ffff80001012265c: ldr x3, [x3, #2480] : prev_eff_load *= capacity_of(this_cpu); 2.23 : ffff800010122660: ldr x4, [x5, #2480] : prev_eff_load += 1; 0.00 : ffff800010122664: and x7, x7, #0x1 : this_eff_load *= 100; 0.00 : ffff800010122668: add x1, x6, x1, lsl #3 : prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 0.00 : ffff80001012266c: mul x0, x0, x2 : this_eff_load *= 100; 0.01 : ffff800010122670: lsl x1, x1, #2 : prev_eff_load += 1; 0.00 : ffff800010122674: madd x0, x0, x4, x7 : this_eff_load *= capacity_of(prev_cpu); 0.00 : ffff800010122678: mul x1, x1, x3 : return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 0.00 : ffff80001012267c: cmp x1, x0 0.17 : ffff800010122680: b.ge ffff80001012225c // b.tcont : wake_affine(): : if (target == nr_cpumask_bits) 0.00 : ffff800010122684: cmp w24, #0x100 0.00 : ffff800010122688: b.eq ffff80001012225c // b.none 0.00 : ffff80001012268c: mov w2, w24 0.00 : ffff800010122690: b ffff800010122260 : wake_affine_weight(): : this_eff_load = cpu_load(cpu_rq(this_cpu)); 0.20 : ffff800010122694: ldr x1, [x29, #184] 0.00 : ffff800010122698: add x1, x1, #0xe80 : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff80001012269c: add x0, x0, x1 : wake_affine_weight(): : this_eff_load = cpu_load(cpu_rq(this_cpu)); 0.04 : ffff8000101226a0: ldr x6, [x0, #288] 0.00 : ffff8000101226a4: b ffff8000101225f4 : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff8000101226a8: ldr x7, [x0, #288] 0.00 : ffff8000101226ac: b ffff8000101225c0 : find_idlest_group_cpu(): : return i; 0.00 : ffff8000101226b0: str w23, [x29, #180] 0.00 : ffff8000101226b4: b ffff80001012245c : find_idlest_cpu(): : if (weight <= tmp->span_weight) 0.00 : ffff8000101226b8: ldr w21, [x29, #180] 0.00 : ffff8000101226bc: b ffff800010122228 : get_current(): 0.00 : ffff8000101226c0: mrs x8, sp_el0 : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff8000101226c4: ldr x7, [x0, #288] : task_cfs_rq(): : return p->se.cfs_rq; 0.00 : ffff8000101226c8: ldr x6, [x8, #320] : task_h_load(): : update_cfs_rq_h_load(cfs_rq); 0.00 : ffff8000101226cc: mov x0, x6 0.00 : ffff8000101226d0: bl ffff80001011c898 : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.00 : ffff8000101226d4: ldr x2, [x6, #280] 0.00 : ffff8000101226d8: ldr x0, [x8, #416] 0.00 : ffff8000101226dc: ldr x1, [x6, #160] 0.00 : ffff8000101226e0: add x1, x1, #0x1 0.00 : ffff8000101226e4: mul x0, x0, x2 : div64_u64(): 0.00 : ffff8000101226e8: udiv x0, x0, x1 : wake_affine_weight(): : if (current_load > this_eff_load) 0.00 : ffff8000101226ec: cmp x7, x0 0.00 : ffff8000101226f0: b.cs ffff8000101225f0 // b.hs, b.nlast 0.00 : ffff8000101226f4: b ffff80001012225c Percent | Source code & Disassembly of vmlinux for cycles (44804 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001017af20 : : sched_clock(): : { : return (cyc * mult) >> shift; : } : : unsigned long long notrace sched_clock(void) : { 1.59 : ffff80001017af20: stp x29, x30, [sp, #-64]! 0.00 : ffff80001017af24: mov x29, sp 0.07 : ffff80001017af28: stp x19, x20, [sp, #16] 0.00 : ffff80001017af2c: adrp x20, ffff800011926000 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001017af30: add x20, x20, #0x180 : sched_clock(): 0.17 : ffff80001017af34: stp x23, x24, [sp, #48] : unsigned int seq; : struct clock_read_data *rd; : : do { : seq = raw_read_seqcount(&cd.seq); : rd = cd.read_data + (seq & 1); 0.00 : ffff80001017af38: add x23, x20, #0x8 : { 0.57 : ffff80001017af3c: stp x21, x22, [sp, #32] : rd = cd.read_data + (seq & 1); 0.02 : ffff80001017af40: mov w24, #0x28 // #40 : __read_once_size(): 0.85 : ffff80001017af44: ldr w22, [x20] : raw_read_seqcount(): : * masking the LSB. Calling code is responsible for handling that. : */ : static inline unsigned raw_read_seqcount(const seqcount_t *s) : { : unsigned ret = READ_ONCE(s->sequence); : smp_rmb(); 0.60 : ffff80001017af48: dmb ishld : sched_clock(): 30.09 : ffff80001017af4c: and w21, w22, #0x1 0.00 : ffff80001017af50: umull x21, w21, w24 0.00 : ffff80001017af54: add x19, x23, x21 : : cyc = (rd->read_sched_clock() - rd->epoch_cyc) & 0.00 : ffff80001017af58: ldr x0, [x19, #24] 0.00 : ffff80001017af5c: blr x0 : rd->sched_clock_mask; : res = rd->epoch_ns + cyc_to_ns(cyc, rd->mult, rd->shift); 46.12 : ffff80001017af60: ldr x3, [x23, x21] 0.05 : ffff80001017af64: ldp w1, w4, [x19, #32] : rd->sched_clock_mask; 0.00 : ffff80001017af68: ldp x6, x5, [x19, #8] : read_seqcount_retry(): : * If the critical section was invalid, it must be ignored (and typically : * retried). : */ : static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) : { : smp_rmb(); 0.05 : ffff80001017af6c: dmb ishld : sched_clock(): : } while (read_seqcount_retry(&cd.seq, seq)); 18.04 : ffff80001017af70: ldr w2, [x20] 0.00 : ffff80001017af74: cmp w2, w22 0.00 : ffff80001017af78: b.ne ffff80001017af44 // b.any : cyc = (rd->read_sched_clock() - rd->epoch_cyc) & 0.00 : ffff80001017af7c: sub x0, x0, x6 : cyc_to_ns(): : return (cyc * mult) >> shift; 0.00 : ffff80001017af80: mov w1, w1 : sched_clock(): : cyc = (rd->read_sched_clock() - rd->epoch_cyc) & 0.00 : ffff80001017af84: and x0, x0, x5 : : return res; : } 0.00 : ffff80001017af88: ldp x19, x20, [sp, #16] : cyc_to_ns(): : return (cyc * mult) >> shift; 0.00 : ffff80001017af8c: mul x0, x0, x1 : sched_clock(): : } 0.00 : ffff80001017af90: ldp x21, x22, [sp, #32] : cyc_to_ns(): : return (cyc * mult) >> shift; 0.00 : ffff80001017af94: lsr x0, x0, x4 : sched_clock(): : } 0.00 : ffff80001017af98: add x0, x0, x3 1.78 : ffff80001017af9c: ldp x23, x24, [sp, #48] 0.00 : ffff80001017afa0: ldp x29, x30, [sp], #64 0.00 : ffff80001017afa4: ret Percent | Source code & Disassembly of vmlinux for cycles (42002 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010455cd0 : : bio_release_pages(): : } : return len; : } : EXPORT_SYMBOL(bio_add_page); : : void bio_release_pages(struct bio *bio, bool mark_dirty) 0.03 : ffff800010455cd0: stp x29, x30, [sp, #-80]! 0.00 : ffff800010455cd4: mov x29, sp 0.15 : ffff800010455cd8: stp x23, x24, [sp, #48] 0.00 : ffff800010455cdc: mov x24, x0 0.65 : ffff800010455ce0: str x25, [sp, #64] 0.00 : ffff800010455ce4: and w25, w1, #0xff 0.01 : ffff800010455ce8: stp x19, x20, [sp, #16] : bvec_init_iter_all(): : } : : static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all) : { : iter_all->done = 0; : iter_all->idx = 0; 0.00 : ffff800010455cec: mov w23, #0x0 // #0 : bio_release_pages(): 0.05 : ffff800010455cf0: stp x21, x22, [sp, #32] 0.00 : ffff800010455cf4: mov w20, #0x1000 // #4096 : bvec_init_iter_all(): : iter_all->done = 0; 0.00 : ffff800010455cf8: mov w21, #0x0 // #0 : __ll_sc_atomic_sub_return(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010455cfc: mov w22, #0x1 // #1 : bio_next_segment(): : } : : static inline bool bio_next_segment(const struct bio *bio, : struct bvec_iter_all *iter) : { : if (iter->idx >= bio->bi_vcnt) 0.17 : ffff800010455d00: ldrh w0, [x24, #96] 0.00 : ffff800010455d04: cmp w0, w23 0.00 : ffff800010455d08: b.le ffff800010455d98 : return false; : : bvec_advance(&bio->bi_io_vec[iter->idx], iter); 0.61 : ffff800010455d0c: ldr x4, [x24, #104] 0.00 : ffff800010455d10: sbfiz x0, x23, #4, #32 0.00 : ffff800010455d14: add x3, x4, x0 : bvec_advance(): : static inline void bvec_advance(const struct bio_vec *bvec, : struct bvec_iter_all *iter_all) : { : struct bio_vec *bv = &iter_all->bv; : : if (iter_all->done) { 0.00 : ffff800010455d18: cbz w21, ffff800010455dcc : bv->bv_page++; 0.00 : ffff800010455d1c: add x19, x19, #0x40 0.00 : ffff800010455d20: mov w2, #0x1000 // #4096 : bv->bv_offset = 0; : } else { : bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); : bv->bv_offset = bvec->bv_offset & ~PAGE_MASK; : } : bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset, 0.61 : ffff800010455d24: ldr w3, [x3, #8] 0.19 : ffff800010455d28: sub w0, w3, w21 0.03 : ffff800010455d2c: cmp w0, w2 0.00 : ffff800010455d30: csel w0, w0, w2, ls // ls = plast : bvec->bv_len - iter_all->done); : iter_all->done += bv->bv_len; 0.15 : ffff800010455d34: add w21, w21, w0 : : if (iter_all->done == bvec->bv_len) { 0.01 : ffff800010455d38: cmp w3, w21 0.00 : ffff800010455d3c: b.eq ffff800010455db0 // b.none : bio_release_pages(): : : if (bio_flagged(bio, BIO_NO_PAGE_REF)) : return; : : bio_for_each_segment_all(bvec, bio, iter_all) { : if (mark_dirty && !PageCompound(bvec->bv_page)) 0.63 : ffff800010455d40: cbz w25, ffff800010455d58 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010455d44: ldr x0, [x19] : PageCompound(): : return READ_ONCE(page->compound_head) & 1; : } : : static __always_inline int PageCompound(struct page *page) : { : return test_bit(PG_head, &page->flags) || PageTail(page); 0.00 : ffff800010455d48: tbnz w0, #16, ffff800010455d58 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010455d4c: ldr x0, [x19, #8] : PageCompound(): 0.00 : ffff800010455d50: tbz w0, #0, ffff800010455de8 0.00 : ffff800010455d54: nop : __read_once_size(): 0.21 : ffff800010455d58: ldr x2, [x19, #8] : compound_head(): : return (struct page *) (head - 1); 0.02 : ffff800010455d5c: sub x0, x2, #0x1 0.05 : ffff800010455d60: tst x2, #0x1 0.01 : ffff800010455d64: csel x0, x0, x19, ne // ne = any : page_ref_dec_and_test(): : return ret; : } : : static inline int page_ref_dec_and_test(struct page *page) : { : int ret = atomic_dec_and_test(&page->_refcount); 0.08 : ffff800010455d68: add x3, x0, #0x34 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010455d6c: b ffff800010455dbc 0.32 : ffff800010455d70: b ffff800010455dbc : __lse_atomic_sub_return(): : } : : ATOMIC_OP_SUB_RETURN(_relaxed, ) : ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC_OP_SUB_RETURN(_release, l, "memory") : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.27 : ffff800010455d74: mov w2, w22 0.00 : ffff800010455d78: neg w2, w2 0.19 : ffff800010455d7c: ldaddal w2, w1, [x3] 91.22 : ffff800010455d80: add w2, w2, w1 : put_page(): : if (page_is_devmap_managed(page)) { : put_devmap_managed_page(page); : return; : } : : if (put_page_testzero(page)) 0.00 : ffff800010455d84: cbnz w2, ffff800010455d00 : __put_page(page); 0.00 : ffff800010455d88: bl ffff8000101dfec0 <__put_page> : bio_next_segment(): : if (iter->idx >= bio->bi_vcnt) 0.00 : ffff800010455d8c: ldrh w0, [x24, #96] 0.00 : ffff800010455d90: cmp w0, w23 0.00 : ffff800010455d94: b.gt ffff800010455d0c : bio_release_pages(): : set_page_dirty_lock(bvec->bv_page); : put_page(bvec->bv_page); : } : } 1.78 : ffff800010455d98: ldp x19, x20, [sp, #16] 0.02 : ffff800010455d9c: ldp x21, x22, [sp, #32] 0.00 : ffff800010455da0: ldp x23, x24, [sp, #48] 0.00 : ffff800010455da4: ldr x25, [sp, #64] 0.75 : ffff800010455da8: ldp x29, x30, [sp], #80 0.00 : ffff800010455dac: ret : bvec_advance(): : iter_all->idx++; 0.03 : ffff800010455db0: add w23, w23, #0x1 : iter_all->done = 0; 0.02 : ffff800010455db4: mov w21, #0x0 // #0 0.80 : ffff800010455db8: b ffff800010455d40 : __ll_sc_atomic_sub_return(): 0.00 : ffff800010455dbc: add x3, x0, #0x34 0.00 : ffff800010455dc0: b ffff800010458ad4 : put_page(): : if (put_page_testzero(page)) 0.00 : ffff800010455dc4: cbnz w2, ffff800010455d00 0.00 : ffff800010455dc8: b ffff800010455d88 : bvec_advance(): : bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); 0.01 : ffff800010455dcc: ldr w2, [x3, #12] 0.12 : ffff800010455dd0: ldr x1, [x4, x0] : bv->bv_offset = bvec->bv_offset & ~PAGE_MASK; 0.00 : ffff800010455dd4: and w0, w2, #0xfff : bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); 0.00 : ffff800010455dd8: lsr w19, w2, #12 0.00 : ffff800010455ddc: sub w2, w20, w0 0.79 : ffff800010455de0: add x19, x1, x19, lsl #6 0.01 : ffff800010455de4: b ffff800010455d24 : bio_release_pages(): : set_page_dirty_lock(bvec->bv_page); 0.00 : ffff800010455de8: mov x0, x19 0.00 : ffff800010455dec: bl ffff8000101dc538 0.00 : ffff800010455df0: b ffff800010455d58 Percent | Source code & Disassembly of vmlinux for cycles (38187 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff8000104642f0 : : blk_mq_complete_request(): : * Description: : * Ends all I/O on a request. It does not handle partial completions. : * The actual completion happens out-of-order, through a IPI handler. : **/ : bool blk_mq_complete_request(struct request *rq) : { 1.60 : ffff8000104642f0: stp x29, x30, [sp, #-48]! : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104642f4: mov w1, #0x2 // #2 : blk_mq_complete_request(): 0.00 : ffff8000104642f8: mov x29, sp 0.00 : ffff8000104642fc: stp x20, x21, [sp, #24] : __write_once_size(): 0.00 : ffff800010464300: str w1, [x0, #208] : __blk_mq_complete_request(): : struct blk_mq_ctx *ctx = rq->mq_ctx; 0.65 : ffff800010464304: ldp x20, x21, [x0] : if (q->nr_hw_queues == 1) { 19.83 : ffff800010464308: ldr w1, [x20, #80] 0.25 : ffff80001046430c: cmp w1, #0x1 0.09 : ffff800010464310: b.eq ffff800010464348 // b.none 15.29 : ffff800010464314: str x19, [x29, #16] 0.01 : ffff800010464318: mov x19, x0 : if ((rq->cmd_flags & REQ_HIPRI) || 0.90 : ffff80001046431c: ldr w0, [x0, #24] 0.00 : ffff800010464320: tbz w0, #25, ffff80001046435c : q->mq_ops->complete(rq); 0.00 : ffff800010464324: ldr x1, [x20, #48] 0.00 : ffff800010464328: mov x0, x19 0.00 : ffff80001046432c: ldr x1, [x1, #48] 0.00 : ffff800010464330: blr x1 0.00 : ffff800010464334: ldr x19, [x29, #16] : blk_mq_complete_request(): : if (unlikely(blk_should_fake_timeout(rq->q))) : return false; : __blk_mq_complete_request(rq); : return true; : } 0.17 : ffff800010464338: mov w0, #0x1 // #1 0.00 : ffff80001046433c: ldp x20, x21, [sp, #24] 0.02 : ffff800010464340: ldp x29, x30, [sp], #48 0.13 : ffff800010464344: ret : __blk_mq_complete_request(): : __blk_complete_request(rq); 0.00 : ffff800010464348: bl ffff800010463100 <__blk_complete_request> : blk_mq_complete_request(): : } 0.00 : ffff80001046434c: mov w0, #0x1 // #1 0.00 : ffff800010464350: ldp x20, x21, [sp, #24] 0.00 : ffff800010464354: ldp x29, x30, [sp], #48 0.00 : ffff800010464358: ret : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.01 : ffff80001046435c: ldr x0, [x20, #104] : __blk_mq_complete_request(): : if ((rq->cmd_flags & REQ_HIPRI) || 0.00 : ffff800010464360: tst w0, #0x10 0.00 : ffff800010464364: b.eq ffff800010464324 // b.none 0.31 : ffff800010464368: str x22, [x29, #40] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.73 : ffff80001046436c: mrs x1, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.07 : ffff800010464370: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010464374: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.01 : ffff800010464378: str w0, [x1, #16] : test_bit(): 0.01 : ffff80001046437c: ldr x2, [x20, #104] : __blk_mq_complete_request(): : cpu = get_cpu(); 0.00 : ffff800010464380: adrp x0, ffff80001151d000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.86 : ffff800010464384: mrs x1, tpidr_el1 : __blk_mq_complete_request(): 0.06 : ffff800010464388: add x0, x0, #0x18 : if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags)) 0.00 : ffff80001046438c: tst w2, #0x1000 : cpu = get_cpu(); 0.00 : ffff800010464390: ldr w22, [x0, x1] : shared = cpus_share_cache(cpu, ctx->cpu); 0.03 : ffff800010464394: ldr w1, [x21, #64] : if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags)) 0.18 : ffff800010464398: b.eq ffff8000104643dc // b.none : if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { 0.00 : ffff80001046439c: cmp w22, w1 0.00 : ffff8000104643a0: b.ne ffff8000104643f8 // b.any : q->mq_ops->complete(rq); 0.06 : ffff8000104643a4: ldr x1, [x20, #48] 0.01 : ffff8000104643a8: mov x0, x19 3.51 : ffff8000104643ac: ldr x1, [x1, #48] 0.07 : ffff8000104643b0: blr x1 : get_current(): 1.07 : ffff8000104643b4: mrs x1, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.59 : ffff8000104643b8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000104643bc: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.45 : ffff8000104643c0: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000104643c4: cbz x0, ffff800010464440 : __read_once_size(): : __READ_ONCE_SIZE; 0.17 : ffff8000104643c8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000104643cc: cbz x0, ffff800010464440 6.44 : ffff8000104643d0: ldr x19, [x29, #16] 0.99 : ffff8000104643d4: ldr x22, [x29, #40] 0.00 : ffff8000104643d8: b ffff800010464338 : __blk_mq_complete_request(): : shared = cpus_share_cache(cpu, ctx->cpu); 33.88 : ffff8000104643dc: mov w0, w22 0.00 : ffff8000104643e0: bl ffff800010117770 : if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { 10.52 : ffff8000104643e4: ldr w1, [x21, #64] 0.00 : ffff8000104643e8: tst w0, #0xff 0.00 : ffff8000104643ec: ccmp w22, w1, #0x4, eq // eq = none 0.00 : ffff8000104643f0: b.eq ffff8000104643a4 // b.none 0.00 : ffff8000104643f4: nop : test_bit(): 0.00 : ffff8000104643f8: add w0, w1, #0x3f 0.00 : ffff8000104643fc: cmp w1, #0x0 0.00 : ffff800010464400: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff800010464404: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010464408: add x2, x2, #0x160 0.00 : ffff80001046440c: asr w0, w0, #6 0.00 : ffff800010464410: sxtw x0, w0 0.00 : ffff800010464414: ldr x0, [x2, x0, lsl #3] 0.00 : ffff800010464418: lsr x1, x0, x1 : __blk_mq_complete_request(): 0.00 : ffff80001046441c: tbz w1, #0, ffff8000104643a4 : rq->csd.flags = 0; 0.00 : ffff800010464420: str wzr, [x19, #256] : rq->csd.func = __blk_mq_complete_request_remote; 0.00 : ffff800010464424: adrp x0, ffff800010463000 0.00 : ffff800010464428: add x0, x0, #0xdb8 : rq->csd.info = rq; 0.00 : ffff80001046442c: stp x0, x19, [x19, #240] : smp_call_function_single_async(ctx->cpu, &rq->csd); 0.00 : ffff800010464430: add x1, x19, #0xe8 0.00 : ffff800010464434: ldr w0, [x21, #64] 0.00 : ffff800010464438: bl ffff800010181db0 0.00 : ffff80001046443c: b ffff8000104643b4 : put_cpu(); 0.00 : ffff800010464440: bl ffff800010cd2e50 0.00 : ffff800010464444: ldr x19, [x29, #16] 0.00 : ffff800010464448: ldr x22, [x29, #40] 0.00 : ffff80001046444c: b ffff800010464338 Percent | Source code & Disassembly of vmlinux for cycles (32340 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010cc3d78 <__radix_tree_lookup>: : __radix_tree_lookup(): : * pointing to a node, in which case *@nodep will be NULL. : */ : void *__radix_tree_lookup(const struct radix_tree_root *root, : unsigned long index, struct radix_tree_node **nodep, : void __rcu ***slotp) : { 0.63 : ffff800010cc3d78: mov x8, x0 0.00 : ffff800010cc3d7c: add x9, x0, #0x8 : shift_maxindex(): : return (RADIX_TREE_MAP_SIZE << shift) - 1; 0.00 : ffff800010cc3d80: mov x10, #0x40 // #64 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.11 : ffff800010cc3d84: ldr x0, [x8, #8] : radix_tree_is_internal_node(): : #define RADIX_TREE_ENTRY_MASK 3UL : #define RADIX_TREE_INTERNAL_NODE 2UL : : static inline bool radix_tree_is_internal_node(void *ptr) : { : return ((unsigned long)ptr & RADIX_TREE_ENTRY_MASK) == 0.00 : ffff800010cc3d88: and x4, x0, #0x3 : radix_tree_load_root(): : if (likely(radix_tree_is_internal_node(node))) { 0.00 : ffff800010cc3d8c: cmp x4, #0x2 0.00 : ffff800010cc3d90: b.ne ffff800010cc3e00 <__radix_tree_lookup+0x88> // b.any : entry_to_node(): : return (void *)((unsigned long)ptr & ~RADIX_TREE_INTERNAL_NODE); 0.80 : ffff800010cc3d94: and x5, x0, #0xfffffffffffffffd : node_maxindex(): : return shift_maxindex(node->shift); 0.71 : ffff800010cc3d98: ldrb w5, [x5] : shift_maxindex(): : return (RADIX_TREE_MAP_SIZE << shift) - 1; 0.00 : ffff800010cc3d9c: lsl x5, x10, x5 0.00 : ffff800010cc3da0: sub x5, x5, #0x1 : __radix_tree_lookup(): : : restart: : parent = NULL; : slot = (void __rcu **)&root->xa_head; : radix_tree_load_root(root, &node, &maxindex); : if (index > maxindex) 0.00 : ffff800010cc3da4: cmp x1, x5 12.41 : ffff800010cc3da8: b.hi ffff800010cc3e08 <__radix_tree_lookup+0x90> // b.pmore : slot = (void __rcu **)&root->xa_head; 0.00 : ffff800010cc3dac: mov x7, x9 : parent = NULL; 0.00 : ffff800010cc3db0: mov x5, #0x0 // #0 0.00 : ffff800010cc3db4: b ffff800010cc3de4 <__radix_tree_lookup+0x6c> : entry_to_node(): : return (void *)((unsigned long)ptr & ~RADIX_TREE_INTERNAL_NODE); 0.02 : ffff800010cc3db8: and x5, x0, #0xfffffffffffffffd : radix_tree_descend(): : unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK; 4.20 : ffff800010cc3dbc: ldrb w6, [x5] 0.39 : ffff800010cc3dc0: lsr x4, x1, x6 0.00 : ffff800010cc3dc4: and x4, x4, #0x3f 0.00 : ffff800010cc3dc8: add x4, x4, #0x5 : __radix_tree_lookup(): : while (radix_tree_is_internal_node(node)) { : unsigned offset; : : parent = entry_to_node(node); : offset = radix_tree_descend(parent, &node, index); : slot = parent->slots + offset; 27.73 : ffff800010cc3dcc: add x7, x5, x4, lsl #3 : __read_once_size(): 1.23 : ffff800010cc3dd0: ldr x0, [x5, x4, lsl #3] 0.00 : ffff800010cc3dd4: and x4, x0, #0x3 : __radix_tree_lookup(): : if (node == RADIX_TREE_RETRY) 0.00 : ffff800010cc3dd8: cmp x0, #0x402 0.00 : ffff800010cc3ddc: b.eq ffff800010cc3d84 <__radix_tree_lookup+0xc> // b.none : goto restart; : if (parent->shift == 0) 32.70 : ffff800010cc3de0: cbz w6, ffff800010cc3dec <__radix_tree_lookup+0x74> : while (radix_tree_is_internal_node(node)) { 7.55 : ffff800010cc3de4: cmp x4, #0x2 0.00 : ffff800010cc3de8: b.eq ffff800010cc3db8 <__radix_tree_lookup+0x40> // b.none : break; : } : : if (nodep) 2.72 : ffff800010cc3dec: cbz x2, ffff800010cc3df4 <__radix_tree_lookup+0x7c> : *nodep = parent; 0.00 : ffff800010cc3df0: str x5, [x2] : if (slotp) 2.85 : ffff800010cc3df4: cbz x3, ffff800010cc3dfc <__radix_tree_lookup+0x84> : *slotp = slot; 0.00 : ffff800010cc3df8: str x7, [x3] : return node; : } 5.93 : ffff800010cc3dfc: ret : radix_tree_load_root(): : *maxindex = 0; 0.00 : ffff800010cc3e00: mov x5, #0x0 // #0 0.00 : ffff800010cc3e04: b ffff800010cc3da4 <__radix_tree_lookup+0x2c> : __radix_tree_lookup(): : return NULL; 0.00 : ffff800010cc3e08: mov x0, #0x0 // #0 : } 0.00 : ffff800010cc3e0c: ret Percent | Source code & Disassembly of vmlinux for cycles (28635 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010aeb370 : : arch_counter_read(): : */ : u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; : EXPORT_SYMBOL_GPL(arch_timer_read_counter); : : static u64 arch_counter_read(struct clocksource *cs) : { 2.67 : ffff800010aeb370: stp x29, x30, [sp, #-16]! : return arch_timer_read_counter(); 0.03 : ffff800010aeb374: adrp x0, ffff800011ac4000 : { 0.00 : ffff800010aeb378: mov x29, sp : return arch_timer_read_counter(); 3.87 : ffff800010aeb37c: ldr x0, [x0, #3984] 0.00 : ffff800010aeb380: blr x0 : } 93.42 : ffff800010aeb384: ldp x29, x30, [sp], #16 0.00 : ffff800010aeb388: ret Percent | Source code & Disassembly of vmlinux for cycles (26923 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010253518 : : kmem_cache_free(): : do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr); : } : #endif : : void kmem_cache_free(struct kmem_cache *s, void *x) : { 0.98 : ffff800010253518: stp x29, x30, [sp, #-64]! 0.00 : ffff80001025351c: mov x29, sp 0.18 : ffff800010253520: stp x19, x20, [sp, #16] 0.00 : ffff800010253524: mov x19, x0 0.95 : ffff800010253528: stp x21, x22, [sp, #32] 0.00 : ffff80001025352c: mov x20, x1 0.00 : ffff800010253530: mov x22, x30 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010253534: nop : cache_from_obj(): : * to not do even the assignment. In that case, slab_equal_or_root : * will also be a constant. : */ : if (!memcg_kmem_enabled() && : !IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) && : !unlikely(s->flags & SLAB_CONSISTENCY_CHECKS)) 0.00 : ffff800010253538: ldr w0, [x0, #8] : !IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) && 0.00 : ffff80001025353c: tbnz w0, #8, ffff8000102536fc : kmem_cache_free(): : s = cache_from_obj(s, x); : if (!s) 0.00 : ffff800010253540: cbz x19, ffff80001025357c 0.00 : ffff800010253544: mov x21, #0x1000000000000 // #281474976710656 0.00 : ffff800010253548: add x21, x1, x21 0.00 : ffff80001025354c: mov x0, #0xffffffffffe00000 // #-2097152 0.00 : ffff800010253550: lsr x21, x21, #12 0.00 : ffff800010253554: movk x0, #0xfdff, lsl #32 0.00 : ffff800010253558: add x21, x0, x21, lsl #6 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.69 : ffff80001025355c: ldr x0, [x21, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff800010253560: sub x1, x0, #0x1 0.00 : ffff800010253564: tst x0, #0x1 0.00 : ffff800010253568: csel x21, x1, x21, ne // ne = any : arch_static_branch(): 1.02 : ffff80001025356c: nop : set_freepointer(): : unsigned long freeptr_addr = (unsigned long)object + s->offset; 0.50 : ffff800010253570: ldr w0, [x19, #32] : *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr); 1.28 : ffff800010253574: str xzr, [x20, x0] : slab_free(): : if (slab_free_freelist_hook(s, &head, &tail)) 0.02 : ffff800010253578: cbnz x20, ffff80001025358c : kmem_cache_free(): : return; : slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_); : trace_kmem_cache_free(_RET_IP_, x); : } 2.63 : ffff80001025357c: ldp x19, x20, [sp, #16] 0.03 : ffff800010253580: ldp x21, x22, [sp, #32] 0.03 : ffff800010253584: ldp x29, x30, [sp], #64 0.00 : ffff800010253588: ret 1.64 : ffff80001025358c: str x23, [x29, #48] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.72 : ffff800010253590: mrs x1, sp_el0 : __read_once_size(): 0.46 : ffff800010253594: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010253598: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.20 : ffff80001025359c: str w0, [x1, #16] : do_slab_free(): : tid = this_cpu_read(s->cpu_slab->tid); 0.34 : ffff8000102535a0: ldr x0, [x19] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.77 : ffff8000102535a4: mrs x2, tpidr_el1 : do_slab_free(): 0.47 : ffff8000102535a8: add x0, x0, #0x8 : __read_once_size(): : __READ_ONCE_SIZE; 1.08 : ffff8000102535ac: ldr x23, [x0, x2] 9.22 : ffff8000102535b0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102535b4: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 13.38 : ffff8000102535b8: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102535bc: cbnz x0, ffff8000102536c8 : do_slab_free(): 0.00 : ffff8000102535c0: bl ffff800010cd2e78 : __my_cpu_offset(): 2.86 : ffff8000102535c4: mrs x1, tpidr_el1 : do_slab_free(): : c = raw_cpu_ptr(s->cpu_slab); 0.02 : ffff8000102535c8: ldr x0, [x19] 0.00 : ffff8000102535cc: add x2, x0, x1 : __read_once_size(): : __READ_ONCE_SIZE; 0.08 : ffff8000102535d0: ldr x5, [x2, #8] : do_slab_free(): : unlikely(tid != READ_ONCE(c->tid))); 0.00 : ffff8000102535d4: cmp x5, x23 0.00 : ffff8000102535d8: b.ne ffff800010253590 // b.any : if (likely(page == c->page)) { 1.62 : ffff8000102535dc: ldr x2, [x2, #16] 0.01 : ffff8000102535e0: cmp x2, x21 0.37 : ffff8000102535e4: b.ne ffff800010253784 // b.any : set_freepointer(): : unsigned long freeptr_addr = (unsigned long)object + s->offset; 0.00 : ffff8000102535e8: ldr w2, [x19, #32] : do_slab_free(): : set_freepointer(s, tail_obj, c->freelist); 0.00 : ffff8000102535ec: ldr x4, [x0, x1] : get_current(): 0.00 : ffff8000102535f0: mrs x3, sp_el0 : set_freepointer(): : *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr); 0.00 : ffff8000102535f4: str x4, [x20, x2] : __read_once_size(): 0.02 : ffff8000102535f8: ldr w2, [x3, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102535fc: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.01 : ffff800010253600: str w2, [x3, #16] : do_slab_free(): : if (unlikely(!this_cpu_cmpxchg_double( 0.00 : ffff800010253604: ldr x6, [x0, x1] : next_tid(): : return tid + TID_STEP; 0.00 : ffff800010253608: add x3, x23, #0x100 : __my_cpu_offset(): 0.00 : ffff80001025360c: mrs x0, tpidr_el1 : do_slab_free(): : if (unlikely(!this_cpu_cmpxchg_double( 0.01 : ffff800010253610: ldr x4, [x19] 0.00 : ffff800010253614: add x4, x4, x0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010253618: b ffff8000102536d4 0.00 : ffff80001025361c: b ffff8000102536d4 : __lse__cmpxchg_double(): : : cl); \ : \ : return x0; \ : } : : __CMPXCHG_DBL( , ) 0.00 : ffff800010253620: mov x0, x6 0.00 : ffff800010253624: mov x1, x5 0.00 : ffff800010253628: mov x2, x20 0.00 : ffff80001025362c: casp x0, x1, x2, x3, [x4] 0.25 : ffff800010253630: eor x0, x0, x6 0.00 : ffff800010253634: eor x1, x1, x5 0.00 : ffff800010253638: orr x0, x0, x1 0.00 : ffff80001025363c: mov x23, x0 : get_current(): 0.00 : ffff800010253640: mrs x0, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010253644: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010253648: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001025364c: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010253650: cbnz x1, ffff8000102536ec : do_slab_free(): 0.00 : ffff800010253654: bl ffff800010cd2e78 0.00 : ffff800010253658: cbnz x23, ffff800010253590 0.00 : ffff80001025365c: ldr x23, [x29, #48] : kmem_cache_free(): : } 0.02 : ffff800010253660: ldp x19, x20, [sp, #16] 0.00 : ffff800010253664: ldp x21, x22, [sp, #32] 0.00 : ffff800010253668: ldp x29, x30, [sp], #64 0.00 : ffff80001025366c: ret : slab_want_init_on_free(): : } : : static inline bool slab_want_init_on_free(struct kmem_cache *c) : { : if (static_branch_unlikely(&init_on_free)) : return !(c->ctor || 0.00 : ffff800010253670: ldr x0, [x19, #64] 0.00 : ffff800010253674: cbnz x0, ffff800010253570 : (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))); 0.00 : ffff800010253678: ldr w0, [x19, #8] 0.00 : ffff80001025367c: and w0, w0, #0xff800 0.00 : ffff800010253680: and w0, w0, #0xfff80fff : return !(c->ctor || 0.00 : ffff800010253684: cbnz w0, ffff800010253570 : slab_free_freelist_hook(): : memset(object, 0, s->object_size); 0.00 : ffff800010253688: ldr w2, [x19, #28] 0.00 : ffff80001025368c: mov w1, #0x0 // #0 0.00 : ffff800010253690: mov x0, x20 0.00 : ffff800010253694: bl ffff800010cb80c0 <__memset> : rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad 0.00 : ffff800010253698: ldr w0, [x19, #8] : : 0; 0.00 : ffff80001025369c: mov w3, #0x0 // #0 0.00 : ffff8000102536a0: tbz w0, #10, ffff8000102536a8 : rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad 0.00 : ffff8000102536a4: ldr w3, [x19, #80] : memset((char *)object + s->inuse, 0, 0.00 : ffff8000102536a8: ldr w0, [x19, #72] 0.00 : ffff8000102536ac: mov w1, #0x0 // #0 : s->size - s->inuse - rsize); 0.00 : ffff8000102536b0: ldr w2, [x19, #24] 0.00 : ffff8000102536b4: sub w2, w2, w0 : memset((char *)object + s->inuse, 0, 0.00 : ffff8000102536b8: add x0, x20, w0, uxtw 0.00 : ffff8000102536bc: sub w2, w2, w3 0.00 : ffff8000102536c0: bl ffff800010cb80c0 <__memset> 0.00 : ffff8000102536c4: b ffff800010253570 : __read_once_size(): : __READ_ONCE_SIZE; 3.13 : ffff8000102536c8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102536cc: cbz x0, ffff8000102535c0 28.58 : ffff8000102536d0: b ffff8000102535c4 : __ll_sc__cmpxchg_double(): : : cl); \ : \ : return ret; \ : } : : __CMPXCHG_DBL( , , , ) 0.00 : ffff8000102536d4: b ffff800010255be8 : get_current(): 0.00 : ffff8000102536d8: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff8000102536dc: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102536e0: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102536e4: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102536e8: cbz x1, ffff800010253654 : __read_once_size(): : __READ_ONCE_SIZE; 0.01 : ffff8000102536ec: ldr x0, [x0, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102536f0: cbnz x0, ffff800010253658 : do_slab_free(): : if (unlikely(!this_cpu_cmpxchg_double( 0.00 : ffff8000102536f4: bl ffff800010cd2e78 0.00 : ffff8000102536f8: b ffff800010253658 : virt_to_head_page(): : } : #endif : : static inline struct page *virt_to_head_page(const void *x) : { : struct page *page = virt_to_page(x); 1.50 : ffff8000102536fc: mov x21, #0x1000000000000 // #281474976710656 0.00 : ffff800010253700: add x21, x20, x21 0.00 : ffff800010253704: mov x0, #0xffffffffffe00000 // #-2097152 0.00 : ffff800010253708: lsr x21, x21, #12 0.23 : ffff80001025370c: movk x0, #0xfdff, lsl #32 0.00 : ffff800010253710: add x21, x0, x21, lsl #6 : __read_once_size(): 0.66 : ffff800010253714: ldr x1, [x21, #8] : compound_head(): 0.00 : ffff800010253718: sub x0, x1, #0x1 0.00 : ffff80001025371c: tst x1, #0x1 0.00 : ffff800010253720: csel x0, x0, x21, ne // ne = any : __read_once_size(): 5.94 : ffff800010253724: ldr x2, [x0, #8] : compound_head(): 0.00 : ffff800010253728: sub x1, x2, #0x1 0.00 : ffff80001025372c: tst x2, #0x1 0.00 : ffff800010253730: csel x1, x1, x0, ne // ne = any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 6.25 : ffff800010253734: ldr x1, [x1] : virt_to_cache(): : if (WARN_ONCE(!PageSlab(page), "%s: Object is not a Slab page!\n", 0.00 : ffff800010253738: tst w1, #0x200 0.00 : ffff80001025373c: b.eq ffff8000102537a8 // b.none 4.12 : ffff800010253740: str x23, [x29, #48] : return page->slab_cache; 0.99 : ffff800010253744: ldr x23, [x0, #24] : cache_from_obj(): : WARN_ONCE(cachep && !slab_equal_or_root(cachep, s), 0.00 : ffff800010253748: cbz x23, ffff80001025365c : slab_equal_or_root(): : return p == s || p == s->memcg_params.root_cache; 0.57 : ffff80001025374c: cmp x19, x23 0.00 : ffff800010253750: b.eq ffff80001025377c // b.none 0.00 : ffff800010253754: ldr x0, [x23, #208] 0.00 : ffff800010253758: cmp x19, x0 0.00 : ffff80001025375c: b.eq ffff800010253770 // b.none : cache_from_obj(): : WARN_ONCE(cachep && !slab_equal_or_root(cachep, s), 0.00 : ffff800010253760: adrp x4, ffff800011ad8000 0.00 : ffff800010253764: add x4, x4, #0x4e4 0.00 : ffff800010253768: ldrb w0, [x4, #1] 0.00 : ffff80001025376c: cbz w0, ffff8000102537dc 0.00 : ffff800010253770: mov x19, x23 0.00 : ffff800010253774: ldr x23, [x29, #48] 0.00 : ffff800010253778: b ffff80001025355c 1.60 : ffff80001025377c: ldr x23, [x29, #48] 0.00 : ffff800010253780: b ffff80001025355c : do_slab_free(): : __slab_free(s, page, head, tail_obj, cnt, addr); 2.65 : ffff800010253784: mov x5, x22 0.00 : ffff800010253788: mov w4, #0x1 // #1 0.00 : ffff80001025378c: mov x3, x20 0.00 : ffff800010253790: mov x2, x20 0.77 : ffff800010253794: mov x1, x21 0.00 : ffff800010253798: mov x0, x19 0.00 : ffff80001025379c: bl ffff800010252740 <__slab_free> 0.16 : ffff8000102537a0: ldr x23, [x29, #48] 0.00 : ffff8000102537a4: b ffff80001025357c : virt_to_cache(): : if (WARN_ONCE(!PageSlab(page), "%s: Object is not a Slab page!\n", 0.00 : ffff8000102537a8: adrp x2, ffff800011ad8000 0.00 : ffff8000102537ac: ldrb w0, [x2, #1252] 0.00 : ffff8000102537b0: cbnz w0, ffff80001025357c 0.00 : ffff8000102537b4: mov w3, #0x1 // #1 0.00 : ffff8000102537b8: adrp x1, ffff800010d17000 0.00 : ffff8000102537bc: add x1, x1, #0x800 0.00 : ffff8000102537c0: strb w3, [x2, #1252] 0.00 : ffff8000102537c4: add x1, x1, #0x18 0.00 : ffff8000102537c8: adrp x0, ffff8000111d2000 0.00 : ffff8000102537cc: add x0, x0, #0x9c8 0.00 : ffff8000102537d0: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff8000102537d4: brk #0x800 0.00 : ffff8000102537d8: b ffff80001025357c : cache_from_obj(): : WARN_ONCE(cachep && !slab_equal_or_root(cachep, s), 0.00 : ffff8000102537dc: ldr x2, [x19, #88] 0.00 : ffff8000102537e0: mov w5, #0x1 // #1 0.00 : ffff8000102537e4: ldr x3, [x23, #88] 0.00 : ffff8000102537e8: adrp x1, ffff800010d17000 0.00 : ffff8000102537ec: add x1, x1, #0x800 0.00 : ffff8000102537f0: strb w5, [x4, #1] 0.00 : ffff8000102537f4: add x1, x1, #0x28 0.00 : ffff8000102537f8: adrp x0, ffff8000111d2000 0.00 : ffff8000102537fc: add x0, x0, #0x9e8 0.00 : ffff800010253800: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010253804: brk #0x800 0.00 : ffff800010253808: mov x19, x23 0.00 : ffff80001025380c: ldr x23, [x29, #48] 0.00 : ffff800010253810: b ffff80001025355c Percent | Source code & Disassembly of vmlinux for cycles (26886 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001045c0c8 : : blk_account_io_done(): : /* : * Account IO completion. flush_rq isn't accounted as a : * normal IO on queueing nor completion. Accounting the : * containing request is enough. : */ : if (req->part && blk_do_io_stat(req) && 0.01 : ffff80001045c0c8: ldr x2, [x0, #168] 0.00 : ffff80001045c0cc: cbz x2, ffff80001045c318 : blk_do_io_stat(): : * a) it's attached to a gendisk, and : * b) the queue had IO stats enabled when this request was started : */ : static inline bool blk_do_io_stat(struct request *rq) : { : return rq->rq_disk && (rq->rq_flags & RQF_IO_STAT); 0.10 : ffff80001045c0d0: ldr x2, [x0, #160] 0.00 : ffff80001045c0d4: cbz x2, ffff80001045c318 : blk_account_io_done(): 0.87 : ffff80001045c0d8: ldr w2, [x0, #28] : blk_do_io_stat(): 0.00 : ffff80001045c0dc: tbz w2, #13, ffff80001045c314 : blk_account_io_done(): 0.43 : ffff80001045c0e0: tbnz w2, #4, ffff80001045c370 : { 0.01 : ffff80001045c0e4: stp x29, x30, [sp, #-64]! 0.00 : ffff80001045c0e8: mov x29, sp 0.00 : ffff80001045c0ec: stp x19, x20, [sp, #16] 0.00 : ffff80001045c0f0: mov x19, x0 0.90 : ffff80001045c0f4: stp x21, x22, [sp, #32] 0.00 : ffff80001045c0f8: mov x21, x1 0.30 : ffff80001045c0fc: stp x23, x24, [sp, #48] : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 0.00 : ffff80001045c100: mov w22, #0x2 // #2 : blk_account_io_done(): : !(req->rq_flags & RQF_FLUSH_SEQ)) { : const int sgrp = op_stat_group(req_op(req)); 0.01 : ffff80001045c104: ldr w0, [x0, #24] 0.00 : ffff80001045c108: and w1, w0, #0xff : op_is_write(): 0.00 : ffff80001045c10c: and w0, w0, #0x1 0.00 : ffff80001045c110: cmp w1, #0x3 0.04 : ffff80001045c114: csel w22, w0, w22, ne // ne = any : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001045c118: bl ffff80001015c1b0 <__rcu_read_lock> : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.03 : ffff80001045c11c: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.78 : ffff80001045c120: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001045c124: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.36 : ffff80001045c128: str w0, [x1, #16] : blk_account_io_done(): : struct hd_struct *part; : : part_stat_lock(); : part = req->part; : : update_io_ticks(part, jiffies); 0.00 : ffff80001045c12c: adrp x0, ffff800011907000 : part = req->part; 0.04 : ffff80001045c130: ldr x20, [x19, #168] : part_stat_inc(part, ios[sgrp]); 0.00 : ffff80001045c134: adrp x23, ffff80001151d000 0.00 : ffff80001045c138: adrp x24, ffff800011909000 : update_io_ticks(part, jiffies); 0.01 : ffff80001045c13c: ldr x1, [x0, #2432] 0.00 : ffff80001045c140: mov x0, x20 0.00 : ffff80001045c144: bl ffff800010458638 : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.38 : ffff80001045c148: mov x4, sp : blk_account_io_done(): : part_stat_inc(part, ios[sgrp]); 0.00 : ffff80001045c14c: add x0, x23, #0x18 0.00 : ffff80001045c150: add x1, x24, #0x928 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.12 : ffff80001045c154: mrs x2, tpidr_el1 : blk_account_io_done(): 0.00 : ffff80001045c158: ldrsw x3, [x0, x2] 0.00 : ffff80001045c15c: sxtw x0, w22 2.25 : ffff80001045c160: ldr x2, [x20, #840] 0.00 : ffff80001045c164: lsl x5, x0, #3 0.27 : ffff80001045c168: ldr x1, [x1, x3, lsl #3] 0.00 : ffff80001045c16c: add x2, x2, x1 0.00 : ffff80001045c170: add x2, x2, x5 2.15 : ffff80001045c174: ldr x3, [x2, #64] 0.00 : ffff80001045c178: add x3, x3, #0x1 10.78 : ffff80001045c17c: str x3, [x2, #64] 2.32 : ffff80001045c180: ldr w1, [x20, #820] 0.00 : ffff80001045c184: cbz w1, ffff80001045c1c0 : part_to_disk(): : struct lockdep_map lockdep_map; : }; : : static inline struct gendisk *part_to_disk(struct hd_struct *part) : { : if (likely(part)) { 0.04 : ffff80001045c188: cbz x20, ffff80001045c3c4 : if (part->partno) : return dev_to_disk(part_to_dev(part)->parent); 0.00 : ffff80001045c18c: ldr x1, [x20, #104] 0.00 : ffff80001045c190: sub x1, x1, #0x70 : __my_cpu_offset(): 0.53 : ffff80001045c194: mrs x6, tpidr_el1 : blk_account_io_done(): 0.68 : ffff80001045c198: add x2, x23, #0x18 0.00 : ffff80001045c19c: add x3, x24, #0x928 0.03 : ffff80001045c1a0: ldrsw x6, [x2, x6] 0.00 : ffff80001045c1a4: ldr x1, [x1, #912] 0.62 : ffff80001045c1a8: ldr x2, [x3, x6, lsl #3] 0.00 : ffff80001045c1ac: add x1, x1, x2 0.00 : ffff80001045c1b0: add x1, x1, x5 0.65 : ffff80001045c1b4: ldr x2, [x1, #64] 0.00 : ffff80001045c1b8: add x2, x2, #0x1 1.33 : ffff80001045c1bc: str x2, [x1, #64] : __my_cpu_offset(): 0.10 : ffff80001045c1c0: mrs x4, tpidr_el1 : blk_account_io_done(): : part_stat_add(part, nsecs[sgrp], now - req->start_time_ns); 0.39 : ffff80001045c1c4: add x2, x23, #0x18 0.00 : ffff80001045c1c8: add x3, x24, #0x928 0.48 : ffff80001045c1cc: ldrsw x4, [x2, x4] 0.32 : ffff80001045c1d0: ldr x1, [x20, #840] 0.02 : ffff80001045c1d4: ldr x2, [x19, #176] 0.36 : ffff80001045c1d8: ldr x4, [x3, x4, lsl #3] 0.00 : ffff80001045c1dc: sub x3, x21, x2 0.00 : ffff80001045c1e0: add x1, x1, x4 0.54 : ffff80001045c1e4: ldr x2, [x1, x0, lsl #3] 0.00 : ffff80001045c1e8: add x2, x2, x3 0.91 : ffff80001045c1ec: str x2, [x1, x0, lsl #3] 0.12 : ffff80001045c1f0: ldr w1, [x20, #820] 0.00 : ffff80001045c1f4: cbz w1, ffff80001045c234 : part_to_disk(): : if (likely(part)) { 0.31 : ffff80001045c1f8: cbz x20, ffff80001045c3bc : return dev_to_disk(part_to_dev(part)->parent); 0.33 : ffff80001045c1fc: ldr x1, [x20, #104] 0.00 : ffff80001045c200: sub x1, x1, #0x70 : __my_cpu_offset(): 0.50 : ffff80001045c204: mrs x4, tpidr_el1 : blk_account_io_done(): 0.04 : ffff80001045c208: add x2, x23, #0x18 0.00 : ffff80001045c20c: add x3, x24, #0x928 0.26 : ffff80001045c210: ldrsw x4, [x2, x4] 0.42 : ffff80001045c214: ldr x1, [x1, #912] 0.56 : ffff80001045c218: ldr x2, [x19, #176] 0.02 : ffff80001045c21c: ldr x4, [x3, x4, lsl #3] 0.00 : ffff80001045c220: sub x3, x21, x2 0.01 : ffff80001045c224: add x1, x1, x4 0.32 : ffff80001045c228: ldr x2, [x1, x0, lsl #3] 0.00 : ffff80001045c22c: add x2, x2, x3 1.05 : ffff80001045c230: str x2, [x1, x0, lsl #3] : part_stat_add(part, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns)); 0.36 : ffff80001045c234: ldr x0, [x19, #176] 0.00 : ffff80001045c238: sub x0, x21, x0 0.00 : ffff80001045c23c: bl ffff800010167320 0.54 : ffff80001045c240: add x2, x23, #0x18 0.00 : ffff80001045c244: add x3, x24, #0x928 : __my_cpu_offset(): 0.38 : ffff80001045c248: mrs x4, tpidr_el1 : blk_account_io_done(): 0.03 : ffff80001045c24c: ldrsw x4, [x2, x4] 0.34 : ffff80001045c250: ldr x1, [x20, #840] 0.65 : ffff80001045c254: ldr x2, [x3, x4, lsl #3] 0.00 : ffff80001045c258: add x1, x1, x2 0.77 : ffff80001045c25c: ldr x2, [x1, #136] 0.00 : ffff80001045c260: add x0, x2, x0 1.66 : ffff80001045c264: str x0, [x1, #136] 0.28 : ffff80001045c268: ldr w0, [x20, #820] 0.00 : ffff80001045c26c: cbnz w0, ffff80001045c31c : part_dec_in_flight(req->q, part, rq_data_dir(req)); 0.59 : ffff80001045c270: ldr w2, [x19, #24] 0.00 : ffff80001045c274: mov x1, x20 0.08 : ffff80001045c278: ldr x0, [x19] 0.00 : ffff80001045c27c: and w2, w2, #0x1 0.00 : ffff80001045c280: bl ffff8000104706c8 : rcu_read_lock(): 1.12 : ffff80001045c284: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : __READ_ONCE_SIZE; 1.78 : ffff80001045c288: ldr x0, [x20, #856] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001045c28c: tst x0, #0x3 0.00 : ffff80001045c290: b.ne ffff80001045c380 // b.any : get_current(): 0.15 : ffff80001045c294: mrs x1, sp_el0 : __read_once_size(): 0.09 : ffff80001045c298: ldr w2, [x1, #16] : __preempt_count_add(): 0.00 : ffff80001045c29c: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.78 : ffff80001045c2a0: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.59 : ffff80001045c2a4: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.38 : ffff80001045c2a8: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.07 : ffff80001045c2ac: add x0, x0, x2 0.22 : ffff80001045c2b0: ldxr x5, [x0] 14.40 : ffff80001045c2b4: add x5, x5, x3 0.04 : ffff80001045c2b8: stxr w4, x5, [x0] 0.00 : ffff80001045c2bc: cbnz w4, ffff80001045c2b0 : __read_once_size(): : __READ_ONCE_SIZE; 0.29 : ffff80001045c2c0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.01 : ffff80001045c2c4: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.55 : ffff80001045c2c8: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045c2cc: cbz x0, ffff80001045c368 : __read_once_size(): : __READ_ONCE_SIZE; 0.11 : ffff80001045c2d0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045c2d4: cbz x0, ffff80001045c368 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 4.32 : ffff80001045c2d8: bl ffff800010160d20 <__rcu_read_unlock> : get_current(): 1.69 : ffff80001045c2dc: mrs x1, sp_el0 : __read_once_size(): 0.64 : ffff80001045c2e0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.01 : ffff80001045c2e4: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.55 : ffff80001045c2e8: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045c2ec: cbz x0, ffff80001045c2f8 : __read_once_size(): : __READ_ONCE_SIZE; 0.22 : ffff80001045c2f0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045c2f4: cbnz x0, ffff80001045c2fc : blk_account_io_done(): : : hd_struct_put(part); : part_stat_unlock(); 0.00 : ffff80001045c2f8: bl ffff800010cd2e50 : rcu_read_unlock(): 21.17 : ffff80001045c2fc: bl ffff800010160d20 <__rcu_read_unlock> : blk_account_io_done(): : } : } 0.02 : ffff80001045c300: ldp x19, x20, [sp, #16] 1.18 : ffff80001045c304: ldp x21, x22, [sp, #32] 0.01 : ffff80001045c308: ldp x23, x24, [sp, #48] 0.13 : ffff80001045c30c: ldp x29, x30, [sp], #64 0.00 : ffff80001045c310: ret 0.00 : ffff80001045c314: ret 0.00 : ffff80001045c318: ret : part_stat_add(part, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns)); 0.36 : ffff80001045c31c: ldr x0, [x19, #176] 0.00 : ffff80001045c320: sub x0, x21, x0 0.07 : ffff80001045c324: bl ffff800010167320 : part_to_disk(): : if (likely(part)) { 0.20 : ffff80001045c328: cbz x20, ffff80001045c3cc : if (part->partno) 0.31 : ffff80001045c32c: ldr w1, [x20, #820] 0.00 : ffff80001045c330: cbnz w1, ffff80001045c374 : else : return dev_to_disk(part_to_dev(part)); 0.00 : ffff80001045c334: sub x2, x20, #0x48 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.45 : ffff80001045c338: mrs x3, tpidr_el1 : blk_account_io_done(): 0.27 : ffff80001045c33c: add x1, x23, #0x18 0.00 : ffff80001045c340: add x24, x24, #0x928 0.37 : ffff80001045c344: ldrsw x3, [x1, x3] 0.61 : ffff80001045c348: ldr x2, [x2, #912] 0.00 : ffff80001045c34c: mov x1, x2 0.51 : ffff80001045c350: ldr x2, [x24, x3, lsl #3] 0.00 : ffff80001045c354: add x1, x1, x2 1.76 : ffff80001045c358: ldr x2, [x1, #136] 0.00 : ffff80001045c35c: add x0, x2, x0 3.47 : ffff80001045c360: str x0, [x1, #136] 0.00 : ffff80001045c364: b ffff80001045c270 : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff80001045c368: bl ffff800010cd2e78 0.00 : ffff80001045c36c: b ffff80001045c2d8 : blk_account_io_done(): 0.00 : ffff80001045c370: ret : part_to_disk(): : return dev_to_disk(part_to_dev(part)->parent); 0.24 : ffff80001045c374: ldr x2, [x20, #104] 0.05 : ffff80001045c378: sub x2, x2, #0x70 0.00 : ffff80001045c37c: b ffff80001045c338 : hd_struct_put(): : return percpu_ref_tryget_live(&part->ref); : } : : static inline void hd_struct_put(struct hd_struct *part) : { : percpu_ref_put(&part->ref); 0.00 : ffff80001045c380: add x0, x20, #0x350 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001045c384: b ffff80001045c3ac 0.00 : ffff80001045c388: b ffff80001045c3ac : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001045c38c: mov x1, #0x1 // #1 0.00 : ffff80001045c390: neg x1, x1 0.00 : ffff80001045c394: ldaddal x1, x2, [x0] 0.00 : ffff80001045c398: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001045c39c: cbnz x1, ffff80001045c2d8 : ref->release(ref); 0.00 : ffff80001045c3a0: ldr x1, [x0, #16] 0.00 : ffff80001045c3a4: blr x1 0.00 : ffff80001045c3a8: b ffff80001045c2d8 : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff80001045c3ac: mov x2, #0x1 // #1 0.00 : ffff80001045c3b0: add x4, x20, #0x350 0.00 : ffff80001045c3b4: b ffff80001045ced0 0.00 : ffff80001045c3b8: b ffff80001045c39c : part_to_disk(): : return NULL; 0.00 : ffff80001045c3bc: mov x1, #0x0 // #0 0.00 : ffff80001045c3c0: b ffff80001045c204 0.00 : ffff80001045c3c4: mov x1, #0x0 // #0 0.00 : ffff80001045c3c8: b ffff80001045c194 0.00 : ffff80001045c3cc: mov x2, #0x0 // #0 0.00 : ffff80001045c3d0: b ffff80001045c338 Percent | Source code & Disassembly of vmlinux for cycles (24809 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001045bc00 : : blk_account_io_completion(): : } : EXPORT_SYMBOL_GPL(blk_rq_err_bytes); : : void blk_account_io_completion(struct request *req, unsigned int bytes) : { : if (req->part && blk_do_io_stat(req)) { 0.92 : ffff80001045bc00: ldr x2, [x0, #168] 0.00 : ffff80001045bc04: cbz x2, ffff80001045bd0c : blk_do_io_stat(): : * a) it's attached to a gendisk, and : * b) the queue had IO stats enabled when this request was started : */ : static inline bool blk_do_io_stat(struct request *rq) : { : return rq->rq_disk && (rq->rq_flags & RQF_IO_STAT); 16.48 : ffff80001045bc08: ldr x2, [x0, #160] 0.00 : ffff80001045bc0c: cbz x2, ffff80001045bd0c 1.26 : ffff80001045bc10: ldr w2, [x0, #28] 0.00 : ffff80001045bc14: tbz w2, #13, ffff80001045bd10 : blk_account_io_completion(): : { 0.41 : ffff80001045bc18: stp x29, x30, [sp, #-48]! : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 0.00 : ffff80001045bc1c: mov w2, #0x2 // #2 : blk_account_io_completion(): 0.00 : ffff80001045bc20: mov x29, sp 0.45 : ffff80001045bc24: stp x19, x20, [sp, #16] 0.00 : ffff80001045bc28: mov x19, x0 0.13 : ffff80001045bc2c: str x21, [sp, #32] 0.00 : ffff80001045bc30: mov w21, w1 : const int sgrp = op_stat_group(req_op(req)); 0.46 : ffff80001045bc34: ldr w0, [x0, #24] 0.00 : ffff80001045bc38: and w1, w0, #0xff : op_is_write(): 0.00 : ffff80001045bc3c: and w0, w0, #0x1 0.00 : ffff80001045bc40: cmp w1, #0x3 0.59 : ffff80001045bc44: csel w20, w0, w2, ne // ne = any : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001045bc48: bl ffff80001015c1b0 <__rcu_read_lock> : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.59 : ffff80001045bc4c: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.17 : ffff80001045bc50: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001045bc54: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.40 : ffff80001045bc58: str w0, [x1, #16] : blk_account_io_completion(): : struct hd_struct *part; : : part_stat_lock(); : part = req->part; : part_stat_add(part, sectors[sgrp], bytes >> 9); 0.00 : ffff80001045bc5c: adrp x3, ffff80001151d000 : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff80001045bc60: mov x7, sp : blk_account_io_completion(): 0.00 : ffff80001045bc64: add x0, x3, #0x18 0.90 : ffff80001045bc68: adrp x6, ffff800011909000 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.31 : ffff80001045bc6c: mrs x1, tpidr_el1 : blk_account_io_completion(): 0.49 : ffff80001045bc70: ldrsw x8, [x0, x1] 0.00 : ffff80001045bc74: add x4, x6, #0x928 : part = req->part; 3.27 : ffff80001045bc78: ldr x5, [x19, #168] 0.00 : ffff80001045bc7c: sbfiz x2, x20, #3, #32 : part_stat_add(part, sectors[sgrp], bytes >> 9); 0.01 : ffff80001045bc80: lsr w1, w21, #9 0.53 : ffff80001045bc84: ldr x4, [x4, x8, lsl #3] 4.69 : ffff80001045bc88: ldr x0, [x5, #840] 0.00 : ffff80001045bc8c: add x0, x0, x4 0.00 : ffff80001045bc90: add x0, x0, x2 7.69 : ffff80001045bc94: ldr x4, [x0, #32] 0.02 : ffff80001045bc98: add x4, x4, x1 21.76 : ffff80001045bc9c: str x4, [x0, #32] 0.39 : ffff80001045bca0: ldr w0, [x5, #820] 0.02 : ffff80001045bca4: cbz w0, ffff80001045bce0 : part_to_disk(): : struct lockdep_map lockdep_map; : }; : : static inline struct gendisk *part_to_disk(struct hd_struct *part) : { : if (likely(part)) { 0.02 : ffff80001045bca8: cbz x5, ffff80001045bd20 : if (part->partno) : return dev_to_disk(part_to_dev(part)->parent); 0.89 : ffff80001045bcac: ldr x0, [x5, #104] 0.00 : ffff80001045bcb0: sub x0, x0, #0x70 : __my_cpu_offset(): 0.69 : ffff80001045bcb4: mrs x4, tpidr_el1 : blk_account_io_completion(): 0.02 : ffff80001045bcb8: add x3, x3, #0x18 0.00 : ffff80001045bcbc: add x6, x6, #0x928 0.00 : ffff80001045bcc0: ldrsw x3, [x3, x4] 0.91 : ffff80001045bcc4: ldr x0, [x0, #912] 6.16 : ffff80001045bcc8: ldr x3, [x6, x3, lsl #3] 0.00 : ffff80001045bccc: add x0, x0, x3 0.00 : ffff80001045bcd0: add x2, x0, x2 0.09 : ffff80001045bcd4: ldr x0, [x2, #32] 0.01 : ffff80001045bcd8: add x1, x0, x1 10.49 : ffff80001045bcdc: str x1, [x2, #32] : get_current(): 0.52 : ffff80001045bce0: mrs x1, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.25 : ffff80001045bce4: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045bce8: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.23 : ffff80001045bcec: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045bcf0: cbnz x0, ffff80001045bd14 : blk_account_io_completion(): : part_stat_unlock(); 0.00 : ffff80001045bcf4: bl ffff800010cd2e50 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.48 : ffff80001045bcf8: bl ffff800010160d20 <__rcu_read_unlock> : blk_account_io_completion(): : } : } 0.38 : ffff80001045bcfc: ldr x21, [sp, #32] 0.47 : ffff80001045bd00: ldp x19, x20, [sp, #16] 0.99 : ffff80001045bd04: ldp x29, x30, [sp], #48 0.00 : ffff80001045bd08: ret 0.00 : ffff80001045bd0c: ret 0.00 : ffff80001045bd10: ret : __read_once_size(): : __READ_ONCE_SIZE; 7.96 : ffff80001045bd14: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045bd18: cbz x0, ffff80001045bcf4 7.49 : ffff80001045bd1c: b ffff80001045bcf8 : part_to_disk(): : else : return dev_to_disk(part_to_dev(part)); : } : return NULL; 0.00 : ffff80001045bd20: mov x0, #0x0 // #0 0.00 : ffff80001045bd24: b ffff80001045bcb4 Percent | Source code & Disassembly of vmlinux for cycles (23175 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001045bd28 : : blk_update_request(): : * %false - this request doesn't have any more data : * %true - this request has more data : **/ : bool blk_update_request(struct request *req, blk_status_t error, : unsigned int nr_bytes) : { 1.02 : ffff80001045bd28: sub sp, sp, #0x50 0.17 : ffff80001045bd2c: stp x29, x30, [sp, #16] 0.00 : ffff80001045bd30: add x29, sp, #0x10 0.16 : ffff80001045bd34: str x20, [sp, #40] 0.02 : ffff80001045bd38: mov w20, w2 0.61 : ffff80001045bd3c: str x22, [sp, #56] 0.02 : ffff80001045bd40: mov x22, x0 1.00 : ffff80001045bd44: str x24, [sp, #72] 0.00 : ffff80001045bd48: and w24, w1, #0xff : blk_status_to_errno(): : if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors))) 0.00 : ffff80001045bd4c: cmp w24, #0xd 0.00 : ffff80001045bd50: b.hi ffff80001045bfb4 // b.pmore : blk_update_request(): : int total_bytes; : : trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes); : : if (!req->bio) 0.24 : ffff80001045bd54: ldr x0, [x22, #56] : return false; 0.00 : ffff80001045bd58: mov w1, #0x0 // #0 : if (!req->bio) 0.00 : ffff80001045bd5c: cbz x0, ffff80001045be14 0.13 : ffff80001045bd60: str x19, [x29, #16] 0.24 : ffff80001045bd64: str x21, [x29, #32] 0.64 : ffff80001045bd68: str x23, [x29, #48] : : #ifdef CONFIG_BLK_DEV_INTEGRITY : if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ && 0.22 : ffff80001045bd6c: ldr w0, [x22, #24] 0.00 : ffff80001045bd70: tbnz w0, #16, ffff80001045be30 : error == BLK_STS_OK) : req->q->integrity.profile->complete_fn(req, nr_bytes); : #endif : : if (unlikely(error && !blk_rq_is_passthrough(req) && 0.26 : ffff80001045bd74: cbnz w24, ffff80001045bfbc : !(req->rq_flags & RQF_QUIET))) : print_req_error(req, error, __func__); : : blk_account_io_completion(req, nr_bytes); 0.57 : ffff80001045bd78: mov w1, w20 0.00 : ffff80001045bd7c: mov x0, x22 : : total_bytes = 0; 0.00 : ffff80001045bd80: mov w23, #0x0 // #0 : blk_account_io_completion(req, nr_bytes); 0.00 : ffff80001045bd84: bl ffff80001045bc00 : while (req->bio) { 1.24 : ffff80001045bd88: b ffff80001045bdbc : req_bio_endio(): : bio_advance(bio, nbytes); 2.91 : ffff80001045bd8c: mov x0, x19 0.00 : ffff80001045bd90: mov w1, w21 0.00 : ffff80001045bd94: bl ffff800010455200 : if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ)) 0.48 : ffff80001045bd98: ldr w0, [x19, #40] 0.00 : ffff80001045bd9c: cbnz w0, ffff80001045bdb0 2.73 : ffff80001045bda0: ldr w0, [x22, #28] 0.00 : ffff80001045bda4: tbnz w0, #4, ffff80001045bdb0 : bio_endio(bio); 0.73 : ffff80001045bda8: mov x0, x19 0.00 : ffff80001045bdac: bl ffff800010456b18 : blk_update_request(): : : /* Completion has already been traced */ : bio_clear_flag(bio, BIO_TRACE_COMPLETION); : req_bio_endio(req, bio, bio_bytes, error); : : total_bytes += bio_bytes; 0.06 : ffff80001045bdb0: add w23, w23, w21 : nr_bytes -= bio_bytes; : : if (!nr_bytes) 0.00 : ffff80001045bdb4: subs w20, w20, w21 0.00 : ffff80001045bdb8: b.eq ffff80001045be88 // b.none : while (req->bio) { 0.38 : ffff80001045bdbc: ldr x19, [x22, #56] 0.01 : ffff80001045bdc0: cbz x19, ffff80001045be58 : unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); 1.36 : ffff80001045bdc4: ldr w0, [x19, #40] 0.00 : ffff80001045bdc8: cmp w20, w0 0.00 : ffff80001045bdcc: csel w21, w20, w0, ls // ls = plast : if (bio_bytes == bio->bi_iter.bi_size) 0.01 : ffff80001045bdd0: b.cc ffff80001045bddc // b.lo, b.ul, b.last : req->bio = bio->bi_next; 58.12 : ffff80001045bdd4: ldr x0, [x19] 18.98 : ffff80001045bdd8: str x0, [x22, #56] : bio_clear_flag(): : bio->bi_flags |= (1U << bit); : } : : static inline void bio_clear_flag(struct bio *bio, unsigned int bit) : { : bio->bi_flags &= ~(1U << bit); 0.51 : ffff80001045bddc: ldrh w0, [x19, #20] 0.00 : ffff80001045bde0: and w0, w0, #0xfffffbff 0.00 : ffff80001045bde4: and w0, w0, #0xffff 1.82 : ffff80001045bde8: strh w0, [x19, #20] : req_bio_endio(): : if (error) 0.00 : ffff80001045bdec: cbz w24, ffff80001045bdf4 : bio->bi_status = error; 0.00 : ffff80001045bdf0: strb w24, [x19, #26] : if (unlikely(rq->rq_flags & RQF_QUIET)) 0.08 : ffff80001045bdf4: ldr w1, [x22, #28] 0.00 : ffff80001045bdf8: tbz w1, #11, ffff80001045bd8c : bio_set_flag(): : bio->bi_flags |= (1U << bit); 0.00 : ffff80001045bdfc: orr w0, w0, #0x40 0.00 : ffff80001045be00: strh w0, [x19, #20] 0.00 : ffff80001045be04: b ffff80001045bd8c 0.00 : ffff80001045be08: ldr x19, [x29, #16] 0.00 : ffff80001045be0c: ldr x21, [x29, #32] 0.00 : ffff80001045be10: ldr x23, [x29, #48] : blk_update_request(): : /* recalculate the number of segments */ : req->nr_phys_segments = blk_recalc_rq_segments(req); : } : : return true; : } 0.00 : ffff80001045be14: mov w0, w1 0.00 : ffff80001045be18: ldr x20, [sp, #40] 0.00 : ffff80001045be1c: ldp x29, x30, [sp, #16] 0.00 : ffff80001045be20: ldr x22, [sp, #56] 0.00 : ffff80001045be24: ldr x24, [sp, #72] 0.00 : ffff80001045be28: add sp, sp, #0x50 0.00 : ffff80001045be2c: ret : if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ && 0.00 : ffff80001045be30: and w1, w0, #0xff 0.00 : ffff80001045be34: cbnz w1, ffff80001045bd74 0.00 : ffff80001045be38: cbnz w24, ffff80001045bfd8 : req->q->integrity.profile->complete_fn(req, nr_bytes); 0.00 : ffff80001045be3c: ldr x2, [x22] 0.00 : ffff80001045be40: mov w1, w20 0.00 : ffff80001045be44: mov x0, x22 0.00 : ffff80001045be48: ldr x2, [x2, #200] 0.00 : ffff80001045be4c: ldr x2, [x2, #24] 0.00 : ffff80001045be50: blr x2 0.00 : ffff80001045be54: b ffff80001045bd78 : req->__data_len = 0; 0.61 : ffff80001045be58: str wzr, [x22, #40] : return false; 0.00 : ffff80001045be5c: mov w1, #0x0 // #0 : } 0.58 : ffff80001045be60: mov w0, w1 : return false; 0.60 : ffff80001045be64: ldr x19, [x29, #16] 0.25 : ffff80001045be68: ldr x21, [x29, #32] 0.20 : ffff80001045be6c: ldr x23, [x29, #48] : } 0.86 : ffff80001045be70: ldp x29, x30, [sp, #16] 0.53 : ffff80001045be74: ldr x20, [sp, #40] 0.15 : ffff80001045be78: ldr x22, [sp, #56] 1.29 : ffff80001045be7c: ldr x24, [sp, #72] 0.00 : ffff80001045be80: add sp, sp, #0x50 0.00 : ffff80001045be84: ret : if (!req->bio) { 0.09 : ffff80001045be88: ldr x4, [x22, #56] 0.12 : ffff80001045be8c: cbz x4, ffff80001045be58 : req->__data_len -= total_bytes; 0.00 : ffff80001045be90: ldr w0, [x22, #40] : blk_rq_is_passthrough(): : return blk_op_is_private(req_op(rq)); : } : : static inline bool blk_rq_is_passthrough(struct request *rq) : { : return blk_rq_is_scsi(rq) || blk_rq_is_private(rq); 0.00 : ffff80001045be94: ldr w2, [x22, #24] : blk_update_request(): 0.00 : ffff80001045be98: sub w0, w0, w23 0.00 : ffff80001045be9c: str w0, [x22, #40] : blk_rq_is_scsi(): : return blk_op_is_scsi(req_op(rq)); 0.00 : ffff80001045bea0: and w1, w2, #0xff : blk_rq_is_passthrough(): : return blk_rq_is_scsi(rq) || blk_rq_is_private(rq); 0.00 : ffff80001045bea4: sub w1, w1, #0x20 0.00 : ffff80001045bea8: cmp w1, #0x3 0.00 : ffff80001045beac: b.ls ffff80001045bec0 // b.plast : blk_update_request(): : req->__sector += total_bytes >> 9; 0.00 : ffff80001045beb0: ldr x1, [x22, #48] 0.00 : ffff80001045beb4: asr w23, w23, #9 0.00 : ffff80001045beb8: add x23, x1, w23, sxtw 0.00 : ffff80001045bebc: str x23, [x22, #48] : if (req->rq_flags & RQF_MIXED_MERGE) { 0.00 : ffff80001045bec0: ldr w3, [x22, #28] 0.00 : ffff80001045bec4: tbz w3, #5, ffff80001045bee0 : req->cmd_flags &= ~REQ_FAILFAST_MASK; 0.00 : ffff80001045bec8: and w2, w2, #0xfffff8ff 0.00 : ffff80001045becc: str w2, [x22, #24] : req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK; 0.00 : ffff80001045bed0: ldr w1, [x4, #16] 0.00 : ffff80001045bed4: and w1, w1, #0x700 0.00 : ffff80001045bed8: orr w2, w1, w2 0.00 : ffff80001045bedc: str w2, [x22, #24] : return true; 0.00 : ffff80001045bee0: mov w1, #0x1 // #1 : if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) { 0.00 : ffff80001045bee4: tbnz w3, #18, ffff80001045be08 : bio_has_data(): : bio->bi_iter.bi_size && 0.00 : ffff80001045bee8: ldr w1, [x4, #40] : if (bio && 0.00 : ffff80001045beec: cbz w1, ffff80001045bf7c : bio_op(bio) != REQ_OP_DISCARD && 0.00 : ffff80001045bef0: ldrb w2, [x4, #16] : bio->bi_iter.bi_size && 0.00 : ffff80001045bef4: sub w3, w2, #0x3 : bio_op(bio) != REQ_OP_SECURE_ERASE && 0.00 : ffff80001045bef8: tst w3, #0xfffffffd 0.00 : ffff80001045befc: ccmp w2, #0x9, #0x4, ne // ne = any 0.00 : ffff80001045bf00: b.ne ffff80001045c078 // b.any : blk_update_request(): : if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) { 0.00 : ffff80001045bf04: cmp w0, w1 0.00 : ffff80001045bf08: b.cs ffff80001045bf7c // b.hs, b.nlast : blk_dump_rq_flags(req, "request botched"); 0.00 : ffff80001045bf0c: mov x0, x22 0.00 : ffff80001045bf10: adrp x1, ffff8000111e9000 0.00 : ffff80001045bf14: add x1, x1, #0xb70 0.00 : ffff80001045bf18: bl ffff80001045a920 : req->__data_len = blk_rq_cur_bytes(req); 0.00 : ffff80001045bf1c: ldr x0, [x22, #56] : blk_rq_cur_bytes(): : return rq->__data_len; : } : : static inline int blk_rq_cur_bytes(const struct request *rq) : { : return rq->bio ? bio_cur_bytes(rq->bio) : 0; 0.00 : ffff80001045bf20: cbz x0, ffff80001045bf78 : bio_has_data(): : bio->bi_iter.bi_size && 0.00 : ffff80001045bf24: ldr w20, [x0, #40] : if (bio && 0.00 : ffff80001045bf28: cbz w20, ffff80001045bf78 : bio_op(bio) != REQ_OP_DISCARD && 0.00 : ffff80001045bf2c: ldrb w1, [x0, #16] : bio->bi_iter.bi_size && 0.00 : ffff80001045bf30: sub w2, w1, #0x3 : bio_op(bio) != REQ_OP_SECURE_ERASE && 0.00 : ffff80001045bf34: tst w2, #0xfffffffd 0.00 : ffff80001045bf38: ccmp w1, #0x9, #0x4, ne // ne = any 0.00 : ffff80001045bf3c: b.eq ffff80001045bf78 // b.none : bio_cur_bytes(): : return bio_iovec(bio).bv_len; 0.00 : ffff80001045bf40: ldp w3, w1, [x0, #44] 0.00 : ffff80001045bf44: mov w2, #0x1000 // #4096 0.00 : ffff80001045bf48: ldr x4, [x0, #104] 0.00 : ffff80001045bf4c: add x3, x4, x3, lsl #4 0.00 : ffff80001045bf50: ldp w0, w3, [x3, #8] 0.00 : ffff80001045bf54: sub w0, w0, w1 0.00 : ffff80001045bf58: add w1, w1, w3 0.00 : ffff80001045bf5c: cmp w0, w20 0.00 : ffff80001045bf60: and w1, w1, #0xfff 0.00 : ffff80001045bf64: sub w1, w2, w1 0.00 : ffff80001045bf68: csel w20, w0, w20, ls // ls = plast 0.00 : ffff80001045bf6c: cmp w1, w20 0.00 : ffff80001045bf70: csel w20, w1, w20, ls // ls = plast 0.00 : ffff80001045bf74: nop : blk_update_request(): 0.00 : ffff80001045bf78: str w20, [x22, #40] : req->nr_phys_segments = blk_recalc_rq_segments(req); 0.00 : ffff80001045bf7c: mov x0, x22 0.00 : ffff80001045bf80: bl ffff8000104618c0 0.00 : ffff80001045bf84: strh w0, [x22, #194] : return true; 0.00 : ffff80001045bf88: mov w1, #0x1 // #1 : } 0.00 : ffff80001045bf8c: mov w0, w1 0.00 : ffff80001045bf90: ldr x19, [x29, #16] 0.00 : ffff80001045bf94: ldr x21, [x29, #32] 0.00 : ffff80001045bf98: ldr x23, [x29, #48] 0.00 : ffff80001045bf9c: ldp x29, x30, [sp, #16] 0.00 : ffff80001045bfa0: ldr x20, [sp, #40] 0.00 : ffff80001045bfa4: ldr x22, [sp, #56] 0.00 : ffff80001045bfa8: ldr x24, [sp, #72] 0.00 : ffff80001045bfac: add sp, sp, #0x50 0.00 : ffff80001045bfb0: ret : blk_status_to_errno(): : if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors))) 0.00 : ffff80001045bfb4: brk #0x800 0.00 : ffff80001045bfb8: b ffff80001045bd54 : blk_rq_is_scsi(): : return blk_op_is_scsi(req_op(rq)); 0.00 : ffff80001045bfbc: and w0, w0, #0xff : blk_op_is_scsi(): : return op == REQ_OP_SCSI_IN || op == REQ_OP_SCSI_OUT; 0.00 : ffff80001045bfc0: sub w1, w0, #0x20 : blk_rq_is_passthrough(): : return blk_rq_is_scsi(rq) || blk_rq_is_private(rq); 0.00 : ffff80001045bfc4: cmp w1, #0x1 0.00 : ffff80001045bfc8: b.ls ffff80001045bd78 // b.plast : blk_op_is_private(): : return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT; 0.00 : ffff80001045bfcc: sub w0, w0, #0x22 : blk_rq_is_passthrough(): : return blk_rq_is_scsi(rq) || blk_rq_is_private(rq); 0.00 : ffff80001045bfd0: cmp w0, #0x1 0.00 : ffff80001045bfd4: b.ls ffff80001045bd78 // b.plast : blk_update_request(): : if (unlikely(error && !blk_rq_is_passthrough(req) && 0.00 : ffff80001045bfd8: ldr w0, [x22, #28] 0.00 : ffff80001045bfdc: tbnz w0, #11, ffff80001045bd78 : print_req_error(): : if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors))) 0.00 : ffff80001045bfe0: cmp w24, #0xd 0.00 : ffff80001045bfe4: b.hi ffff80001045c0bc // b.pmore : printk_ratelimited(KERN_ERR 0.00 : ffff80001045bfe8: adrp x19, ffff800010d3a000 0.00 : ffff80001045bfec: adrp x0, ffff800011938000 0.00 : ffff80001045bff0: add x21, x19, #0xe78 0.00 : ffff80001045bff4: add x0, x0, #0x4d0 0.00 : ffff80001045bff8: add x1, x21, #0x218 0.00 : ffff80001045bffc: add x0, x0, #0x10 0.00 : ffff80001045c000: bl ffff800010cc4388 <___ratelimit> 0.00 : ffff80001045c004: cbz w0, ffff80001045bd78 0.00 : ffff80001045c008: ldr x0, [x22, #160] 0.00 : ffff80001045c00c: add x1, x21, w24, sxtw #4 0.00 : ffff80001045c010: ldr w7, [x22, #24] 0.00 : ffff80001045c014: adrp x3, ffff8000111dd000 0.00 : ffff80001045c018: cmp x0, #0x0 0.00 : ffff80001045c01c: add x3, x3, #0xd68 0.00 : ffff80001045c020: and w5, w7, #0xff 0.00 : ffff80001045c024: add x0, x0, #0xc 0.00 : ffff80001045c028: csel x3, x3, x0, eq // eq = none 0.00 : ffff80001045c02c: ldr x4, [x22, #48] : blk_op_str(): : if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op]) 0.00 : ffff80001045c030: cmp w5, #0x23 : print_req_error(): : printk_ratelimited(KERN_ERR 0.00 : ffff80001045c034: ldr x2, [x1, #296] : blk_op_str(): : if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op]) 0.00 : ffff80001045c038: b.hi ffff80001045c0b0 // b.pmore 0.00 : ffff80001045c03c: mov w0, w5 0.00 : ffff80001045c040: ldr x6, [x21, x0, lsl #3] 0.00 : ffff80001045c044: cbz x6, ffff80001045c0b0 : print_req_error(): : printk_ratelimited(KERN_ERR 0.00 : ffff80001045c048: ldrh w8, [x22, #200] 0.00 : ffff80001045c04c: add x1, x19, #0xe78 0.00 : ffff80001045c050: ldrh w0, [x22, #194] 0.00 : ffff80001045c054: and w7, w7, #0xffffff00 0.00 : ffff80001045c058: str w0, [sp] 0.00 : ffff80001045c05c: add x1, x1, #0x200 0.00 : ffff80001045c060: lsr w8, w8, #13 0.00 : ffff80001045c064: str w8, [sp, #8] 0.00 : ffff80001045c068: adrp x0, ffff8000111e9000 0.00 : ffff80001045c06c: add x0, x0, #0xb18 0.00 : ffff80001045c070: bl ffff80001014a85c 0.00 : ffff80001045c074: b ffff80001045bd78 : bio_cur_bytes(): 0.00 : ffff80001045c078: ldp w5, w2, [x4, #44] 0.00 : ffff80001045c07c: mov w3, #0x1000 // #4096 0.00 : ffff80001045c080: ldr x6, [x4, #104] 0.00 : ffff80001045c084: add x5, x6, x5, lsl #4 0.00 : ffff80001045c088: ldp w4, w5, [x5, #8] 0.00 : ffff80001045c08c: sub w4, w4, w2 0.00 : ffff80001045c090: add w2, w2, w5 0.00 : ffff80001045c094: cmp w4, w1 0.00 : ffff80001045c098: and w2, w2, #0xfff 0.00 : ffff80001045c09c: sub w2, w3, w2 0.00 : ffff80001045c0a0: csel w1, w4, w1, ls // ls = plast 0.00 : ffff80001045c0a4: cmp w2, w1 0.00 : ffff80001045c0a8: csel w1, w2, w1, ls // ls = plast 0.00 : ffff80001045c0ac: b ffff80001045bf04 : blk_op_str(): : const char *op_str = "UNKNOWN"; 0.00 : ffff80001045c0b0: adrp x6, ffff8000111e9000 0.00 : ffff80001045c0b4: add x6, x6, #0x900 0.00 : ffff80001045c0b8: b ffff80001045c048 : print_req_error(): : if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors))) 0.00 : ffff80001045c0bc: brk #0x800 0.00 : ffff80001045c0c0: b ffff80001045bd78 Percent | Source code & Disassembly of vmlinux for cycles (20187 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010458508 : : bio_check_pages_dirty(): : bio_put(bio); : } : } : : void bio_check_pages_dirty(struct bio *bio) : { 0.01 : ffff800010458508: stp x29, x30, [sp, #-48]! : bvec_init_iter_all(): : .bi_bvec_done = 0, \ : } : : static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all) : { : iter_all->done = 0; 0.00 : ffff80001045850c: mov w2, #0x0 // #0 : iter_all->idx = 0; 0.00 : ffff800010458510: mov w4, #0x0 // #0 0.00 : ffff800010458514: mov w8, #0x1000 // #4096 : bio_check_pages_dirty(): 0.24 : ffff800010458518: mov x29, sp 1.48 : ffff80001045851c: str x19, [sp, #16] 0.00 : ffff800010458520: mov x19, x0 : bio_next_segment(): : } : : static inline bool bio_next_segment(const struct bio *bio, : struct bvec_iter_all *iter) : { : if (iter->idx >= bio->bi_vcnt) 0.22 : ffff800010458524: ldrh w7, [x0, #96] : return false; : : bvec_advance(&bio->bi_io_vec[iter->idx], iter); 16.86 : ffff800010458528: sbfiz x0, x4, #4, #32 : if (iter->idx >= bio->bi_vcnt) 0.00 : ffff80001045852c: cmp w7, w4 0.00 : ffff800010458530: b.le ffff800010458608 : bvec_advance(&bio->bi_io_vec[iter->idx], iter); 0.21 : ffff800010458534: ldr x6, [x19, #104] : bvec_advance(): : struct bvec_iter_all *iter_all) : { : struct bio_vec *bv = &iter_all->bv; : : if (iter_all->done) { : bv->bv_page++; 0.00 : ffff800010458538: add x1, x1, #0x40 0.00 : ffff80001045853c: mov w3, #0x1000 // #4096 : bio_next_segment(): 0.01 : ffff800010458540: add x5, x6, x0 : bvec_advance(): : if (iter_all->done) { 0.49 : ffff800010458544: cbnz w2, ffff800010458560 : bv->bv_offset = 0; : } else { : bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); 1.22 : ffff800010458548: ldr x1, [x6, x0] 0.60 : ffff80001045854c: ldr w0, [x5, #12] : bv->bv_offset = bvec->bv_offset & ~PAGE_MASK; 0.00 : ffff800010458550: and w3, w0, #0xfff : bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); 0.00 : ffff800010458554: lsr w0, w0, #12 0.00 : ffff800010458558: sub w3, w8, w3 0.25 : ffff80001045855c: add x1, x1, x0, lsl #6 : } : bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset, 0.14 : ffff800010458560: ldr w5, [x5, #8] 0.00 : ffff800010458564: sub w0, w5, w2 0.00 : ffff800010458568: cmp w0, w3 0.00 : ffff80001045856c: csel w0, w0, w3, ls // ls = plast : bvec->bv_len - iter_all->done); : iter_all->done += bv->bv_len; 2.29 : ffff800010458570: add w2, w2, w0 : : if (iter_all->done == bvec->bv_len) { 0.00 : ffff800010458574: cmp w5, w2 0.00 : ffff800010458578: b.ne ffff800010458584 // b.any : iter_all->idx++; 0.02 : ffff80001045857c: add w4, w4, #0x1 : iter_all->done = 0; 0.01 : ffff800010458580: mov w2, #0x0 // #0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.13 : ffff800010458584: ldr x3, [x1, #8] : compound_head(): : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); : return page; 0.00 : ffff800010458588: sub x0, x3, #0x1 0.00 : ffff80001045858c: tst x3, #0x1 0.00 : ffff800010458590: csel x0, x0, x1, ne // ne = any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 73.02 : ffff800010458594: ldr x0, [x0] : bio_check_pages_dirty(): : struct bio_vec *bvec; : unsigned long flags; : struct bvec_iter_all iter_all; : : bio_for_each_segment_all(bvec, bio, iter_all) { : if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page)) 0.00 : ffff800010458598: tbnz w0, #3, ffff800010458528 : test_bit(): 0.00 : ffff80001045859c: ldr x0, [x1] : PageCompound(): : return READ_ONCE(page->compound_head) & 1; : } : : static __always_inline int PageCompound(struct page *page) : { : return test_bit(PG_head, &page->flags) || PageTail(page); 0.00 : ffff8000104585a0: tbnz w0, #16, ffff800010458528 : __read_once_size(): 0.00 : ffff8000104585a4: ldr x0, [x1, #8] : PageCompound(): 0.00 : ffff8000104585a8: tbnz w0, #0, ffff800010458528 0.00 : ffff8000104585ac: stp x20, x21, [x29, #24] : bio_check_pages_dirty(): : : bio_release_pages(bio, false); : bio_put(bio); : return; : defer: : spin_lock_irqsave(&bio_dirty_lock, flags); 0.00 : ffff8000104585b0: adrp x20, ffff800011b3a000 0.00 : ffff8000104585b4: add x20, x20, #0x2a0 0.00 : ffff8000104585b8: add x21, x20, #0x188 0.00 : ffff8000104585bc: mov x0, x21 0.00 : ffff8000104585c0: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : bio->bi_private = bio_dirty_list; 0.00 : ffff8000104585c4: ldr x2, [x20, #400] : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000104585c8: mov x1, x0 : bio_check_pages_dirty(): 0.00 : ffff8000104585cc: str x2, [x19, #64] : spin_unlock_irqrestore(): 0.00 : ffff8000104585d0: mov x0, x21 : bio_check_pages_dirty(): : bio_dirty_list = bio; 0.00 : ffff8000104585d4: str x19, [x20, #400] : spin_unlock_irqrestore(): 0.00 : ffff8000104585d8: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : schedule_work(): : * queued and leaves it in the same position on the kernel-global : * workqueue otherwise. : */ : static inline bool schedule_work(struct work_struct *work) : { : return queue_work(system_wq, work); 0.00 : ffff8000104585dc: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> : queue_work(): : return queue_work_on(WORK_CPU_UNBOUND, wq, work); 0.00 : ffff8000104585e0: adrp x2, ffff800011938000 0.00 : ffff8000104585e4: add x2, x2, #0x448 0.00 : ffff8000104585e8: ldr x1, [x0, #496] 0.00 : ffff8000104585ec: add x2, x2, #0x20 0.00 : ffff8000104585f0: mov w0, #0x100 // #256 0.00 : ffff8000104585f4: bl ffff800010104e98 : bio_check_pages_dirty(): : spin_unlock_irqrestore(&bio_dirty_lock, flags); : schedule_work(&bio_dirty_work); : } 0.00 : ffff8000104585f8: ldr x19, [sp, #16] 0.00 : ffff8000104585fc: ldp x20, x21, [x29, #24] 0.00 : ffff800010458600: ldp x29, x30, [sp], #48 0.00 : ffff800010458604: ret : bio_release_pages(): : if (bio_flagged(bio, BIO_NO_PAGE_REF)) 1.34 : ffff800010458608: ldrh w0, [x19, #20] 0.00 : ffff80001045860c: tbz w0, #0, ffff800010458624 : bio_check_pages_dirty(): : bio_put(bio); 0.00 : ffff800010458610: mov x0, x19 0.00 : ffff800010458614: bl ffff800010456ab8 : } 0.12 : ffff800010458618: ldr x19, [sp, #16] 0.14 : ffff80001045861c: ldp x29, x30, [sp], #48 0.00 : ffff800010458620: ret : bio_release_pages(): 0.15 : ffff800010458624: mov w1, #0x0 // #0 0.00 : ffff800010458628: mov x0, x19 0.00 : ffff80001045862c: bl ffff800010455cd0 0.03 : ffff800010458630: b ffff800010458610 Percent | Source code & Disassembly of vmlinux for cycles (20979 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010cd7c38 <_raw_spin_unlock>: : _raw_spin_unlock(): : EXPORT_SYMBOL(_raw_spin_lock_bh); : #endif : : #ifdef CONFIG_UNINLINE_SPIN_UNLOCK : void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) : { 0.31 : ffff800010cd7c38: stp x29, x30, [sp, #-16]! : queued_spin_unlock(): : static __always_inline void queued_spin_unlock(struct qspinlock *lock) : { : /* : * unlock() needs release semantics: : */ : smp_store_release(&lock->locked, 0); 0.00 : ffff800010cd7c3c: mov w1, #0x0 // #0 : _raw_spin_unlock(): 0.00 : ffff800010cd7c40: mov x29, sp : queued_spin_unlock(): 4.64 : ffff800010cd7c44: stlrb w1, [x0] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 68.55 : ffff800010cd7c48: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.05 : ffff800010cd7c4c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010cd7c50: sub x0, x0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 3.31 : ffff800010cd7c54: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010cd7c58: cbz x0, ffff800010cd7c64 <_raw_spin_unlock+0x2c> : __read_once_size(): : __READ_ONCE_SIZE; 6.16 : ffff800010cd7c5c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.12 : ffff800010cd7c60: cbnz x0, ffff800010cd7c68 <_raw_spin_unlock+0x30> : __raw_spin_unlock(): : : static inline void __raw_spin_unlock(raw_spinlock_t *lock) : { : spin_release(&lock->dep_map, _RET_IP_); : do_raw_spin_unlock(lock); : preempt_enable(); 0.00 : ffff800010cd7c64: bl ffff800010cd2e50 : _raw_spin_unlock(): : __raw_spin_unlock(lock); : } 16.87 : ffff800010cd7c68: ldp x29, x30, [sp], #16 0.00 : ffff800010cd7c6c: ret Percent | Source code & Disassembly of vmlinux for cycles (17447 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010151a30 : : handle_fasteoi_irq(): : * call when the interrupt has been serviced. This enables support : * for modern forms of interrupt handlers, which handle the flow : * details in hardware, transparently. : */ : void handle_fasteoi_irq(struct irq_desc *desc) : { 72.32 : ffff800010151a30: stp x29, x30, [sp, #-48]! 0.00 : ffff800010151a34: mov x29, sp 1.79 : ffff800010151a38: stp x19, x20, [sp, #16] 0.00 : ffff800010151a3c: mov x19, x0 0.01 : ffff800010151a40: str x21, [sp, #32] : struct irq_chip *chip = desc->irq_data.chip; : : raw_spin_lock(&desc->lock); 0.00 : ffff800010151a44: add x20, x0, #0xd4 0.00 : ffff800010151a48: mov x0, x20 : struct irq_chip *chip = desc->irq_data.chip; 0.01 : ffff800010151a4c: ldr x21, [x19, #112] : raw_spin_lock(&desc->lock); 0.01 : ffff800010151a50: bl ffff800010cd80f0 <_raw_spin_lock> : : if (!irq_may_run(desc)) 1.76 : ffff800010151a54: mov x0, x19 0.00 : ffff800010151a58: bl ffff8000101511d8 1.71 : ffff800010151a5c: tst w0, #0xff 0.00 : ffff800010151a60: b.eq ffff800010151b38 // b.none : goto out; : : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING); 1.42 : ffff800010151a64: ldr w0, [x19, #172] : : /* : * If its disabled or no action available : * then mask it and get out of here: : */ : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) { 1.72 : ffff800010151a68: ldr x2, [x19, #160] : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING); 0.00 : ffff800010151a6c: and w0, w0, #0xffffff3f 0.11 : ffff800010151a70: str w0, [x19, #172] 1.54 : ffff800010151a74: ldr x1, [x19, #104] : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) { 0.00 : ffff800010151a78: cbz x2, ffff800010151b20 : irqd_irq_disabled(): : return __irqd_to_state(d) & IRQD_MOVE_PCNTXT; : } : : static inline bool irqd_irq_disabled(struct irq_data *d) : { : return __irqd_to_state(d) & IRQD_IRQ_DISABLED; 0.02 : ffff800010151a7c: ldr w2, [x1] : handle_fasteoi_irq(): 0.00 : ffff800010151a80: tbnz w2, #16, ffff800010151b20 : __kstat_incr_irqs_this_cpu(): : : #undef __irqd_to_state : : static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc) : { : __this_cpu_inc(*desc->kstat_irqs); 0.48 : ffff800010151a84: ldr x1, [x19, #144] : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff800010151a88: mov x3, sp : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.16 : ffff800010151a8c: mrs x4, tpidr_el1 : __kstat_incr_irqs_this_cpu(): 1.62 : ffff800010151a90: ldr w2, [x1, x4] : __this_cpu_inc(kstat.irqs_sum); 0.06 : ffff800010151a94: adrp x0, ffff80001151f000 0.01 : ffff800010151a98: add x0, x0, #0x10 : __this_cpu_inc(*desc->kstat_irqs); 0.00 : ffff800010151a9c: add w2, w2, #0x1 0.20 : ffff800010151aa0: str w2, [x1, x4] : __my_cpu_offset(): 0.11 : ffff800010151aa4: mrs x2, tpidr_el1 : __kstat_incr_irqs_this_cpu(): : __this_cpu_inc(kstat.irqs_sum); 0.11 : ffff800010151aa8: ldr x1, [x0, x2] 0.00 : ffff800010151aac: add x1, x1, #0x1 1.50 : ffff800010151ab0: str x1, [x0, x2] : kstat_incr_irqs_this_cpu(): : } : : static inline void kstat_incr_irqs_this_cpu(struct irq_desc *desc) : { : __kstat_incr_irqs_this_cpu(desc); : desc->tot_count++; 0.09 : ffff800010151ab4: ldr w0, [x19, #184] : handle_fasteoi_irq(): : mask_irq(desc); : goto out; : } : : kstat_incr_irqs_this_cpu(desc); : if (desc->istate & IRQS_ONESHOT) 1.94 : ffff800010151ab8: ldr w1, [x19, #172] : kstat_incr_irqs_this_cpu(): 0.00 : ffff800010151abc: add w0, w0, #0x1 1.63 : ffff800010151ac0: str w0, [x19, #184] : handle_fasteoi_irq(): 0.00 : ffff800010151ac4: tbz w1, #5, ffff800010151ad4 : irqd_irq_masked(): : } : : static inline bool irqd_irq_masked(struct irq_data *d) : { : return __irqd_to_state(d) & IRQD_IRQ_MASKED; 0.00 : ffff800010151ac8: ldr x0, [x19, #104] 0.00 : ffff800010151acc: ldr w0, [x0] : mask_irq(): : if (irqd_irq_masked(&desc->irq_data)) 0.00 : ffff800010151ad0: tbz w0, #17, ffff800010151b64 : handle_fasteoi_irq(): : mask_irq(desc); : : preflow_handler(desc); : handle_irq_event(desc); 1.47 : ffff800010151ad4: mov x0, x19 0.01 : ffff800010151ad8: bl ffff80001014c630 : cond_unmask_eoi_irq(): : if (!(desc->istate & IRQS_ONESHOT)) { 0.01 : ffff800010151adc: ldr w0, [x19, #172] : handle_fasteoi_irq(): : : cond_unmask_eoi_irq(desc, chip); 0.39 : ffff800010151ae0: ldr x1, [x21, #80] : cond_unmask_eoi_irq(): : if (!(desc->istate & IRQS_ONESHOT)) { 0.00 : ffff800010151ae4: tbz w0, #5, ffff800010151b44 : irqd_irq_disabled(): : return __irqd_to_state(d) & IRQD_IRQ_DISABLED; 0.00 : ffff800010151ae8: ldr x0, [x19, #104] 0.00 : ffff800010151aec: ldr w0, [x0] : cond_unmask_eoi_irq(): : if (!irqd_irq_disabled(&desc->irq_data) && 0.00 : ffff800010151af0: tbnz w0, #16, ffff800010151b70 0.00 : ffff800010151af4: tbz w0, #17, ffff800010151b70 : irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) { 0.00 : ffff800010151af8: ldr x0, [x19, #248] 0.00 : ffff800010151afc: cbnz x0, ffff800010151b70 : chip->irq_eoi(&desc->irq_data); 0.00 : ffff800010151b00: add x0, x19, #0x58 0.00 : ffff800010151b04: blr x1 : irqd_irq_masked(): : return __irqd_to_state(d) & IRQD_IRQ_MASKED; 0.00 : ffff800010151b08: ldr x0, [x19, #104] 0.00 : ffff800010151b0c: ldr w0, [x0] : unmask_irq(): : if (!irqd_irq_masked(&desc->irq_data)) 0.00 : ffff800010151b10: tbz w0, #17, ffff800010151b4c 0.00 : ffff800010151b14: mov x0, x19 0.00 : ffff800010151b18: bl ffff800010151468 0.00 : ffff800010151b1c: b ffff800010151b4c : handle_fasteoi_irq(): : desc->istate |= IRQS_PENDING; 0.00 : ffff800010151b20: orr w0, w0, #0x200 0.00 : ffff800010151b24: str w0, [x19, #172] : irqd_irq_masked(): 0.00 : ffff800010151b28: ldr w0, [x1] : mask_irq(): : if (irqd_irq_masked(&desc->irq_data)) 0.00 : ffff800010151b2c: tbnz w0, #17, ffff800010151b38 0.00 : ffff800010151b30: mov x0, x19 0.00 : ffff800010151b34: bl ffff800010151320 : handle_fasteoi_irq(): : : raw_spin_unlock(&desc->lock); : return; : out: : if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED)) 0.00 : ffff800010151b38: ldr x0, [x21, #280] 0.00 : ffff800010151b3c: tbnz w0, #1, ffff800010151b4c : chip->irq_eoi(&desc->irq_data); 0.00 : ffff800010151b40: ldr x1, [x21, #80] 6.28 : ffff800010151b44: add x0, x19, #0x58 0.00 : ffff800010151b48: blr x1 : raw_spin_unlock(&desc->lock); 0.76 : ffff800010151b4c: mov x0, x20 0.00 : ffff800010151b50: bl ffff800010cd7c38 <_raw_spin_unlock> : } 0.70 : ffff800010151b54: ldp x19, x20, [sp, #16] 0.06 : ffff800010151b58: ldr x21, [sp, #32] 0.00 : ffff800010151b5c: ldp x29, x30, [sp], #48 0.00 : ffff800010151b60: ret : mask_irq(): 0.00 : ffff800010151b64: mov x0, x19 0.00 : ffff800010151b68: bl ffff800010151320 0.00 : ffff800010151b6c: b ffff800010151ad4 : cond_unmask_eoi_irq(): : } else if (!(chip->flags & IRQCHIP_EOI_THREADED)) { 0.00 : ffff800010151b70: ldr x0, [x21, #280] 0.00 : ffff800010151b74: tbz w0, #6, ffff800010151b44 0.00 : ffff800010151b78: b ffff800010151b4c Percent | Source code & Disassembly of vmlinux for cycles (16740 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010134a68 <__update_load_avg_cfs_rq>: : ___update_load_sum(): : ___update_load_sum(u64 now, struct sched_avg *sa, : unsigned long load, unsigned long runnable, int running) : { : u64 delta; : : delta = now - sa->last_update_time; 0.75 : ffff800010134a68: ldr x3, [x1, #128] : /* : * This should only happen when time goes backwards, which it : * unfortunately does during sched clock init when we swap over to TSC. : */ : if ((s64)delta < 0) { 0.01 : ffff800010134a6c: subs x2, x0, x3 0.00 : ffff800010134a70: b.mi ffff800010134b88 <__update_load_avg_cfs_rq+0x120> // b.first : : /* : * Use 1024ns as the unit of measurement since it's a reasonable : * approximation of 1us and fast to compute. : */ : delta >>= 10; 1.37 : ffff800010134a74: lsr x4, x2, #10 : if (!delta) 0.00 : ffff800010134a78: cbz x4, ffff800010134b80 <__update_load_avg_cfs_rq+0x118> : __update_load_avg_cfs_rq(): : : return 0; : } : : int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) : { 1.12 : ffff800010134a7c: stp x29, x30, [sp, #-16]! 0.01 : ffff800010134a80: add x8, x1, #0x80 : ___update_load_sum(): : sa->last_update_time += delta << 10; 0.00 : ffff800010134a84: and x2, x2, #0xfffffffffffffc00 0.00 : ffff800010134a88: mov x7, x1 : __update_load_avg_cfs_rq(): : { 1.83 : ffff800010134a8c: mov x29, sp : ___update_load_sum(): : sa->last_update_time += delta << 10; 0.01 : ffff800010134a90: add x2, x2, x3 0.30 : ffff800010134a94: str x2, [x1, #128] : __update_load_avg_cfs_rq(): : if (___update_load_sum(now, &cfs_rq->avg, 0.78 : ffff800010134a98: ldr x9, [x1] 0.05 : ffff800010134a9c: mov w0, w4 : scale_load_down(cfs_rq->load.weight), : scale_load_down(cfs_rq->runnable_weight), 83.43 : ffff800010134aa0: ldr x10, [x1, #16] 0.60 : ffff800010134aa4: ldr w1, [x8, #28] : if (___update_load_sum(now, &cfs_rq->avg, 0.00 : ffff800010134aa8: lsr x9, x9, #10 : cfs_rq->curr != NULL)) { 0.16 : ffff800010134aac: ldr x2, [x7, #72] 0.00 : ffff800010134ab0: add x4, x4, w1, uxtw 0.00 : ffff800010134ab4: lsr x12, x4, #10 : ___update_load_sum(): : if (!load) 0.00 : ffff800010134ab8: cbnz x9, ffff800010134b94 <__update_load_avg_cfs_rq+0x12c> : runnable = running = 0; 0.37 : ffff800010134abc: mov x10, #0x0 // #0 0.01 : ffff800010134ac0: mov w13, #0x0 // #0 : accumulate_sum(): : if (periods) { 0.01 : ffff800010134ac4: cbz x12, ffff800010134bc0 <__update_load_avg_cfs_rq+0x158> : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.03 : ffff800010134ac8: cmp x12, #0x7e0 : accumulate_sum(): : sa->load_sum = decay_load(sa->load_sum, periods); 0.01 : ffff800010134acc: ldr x3, [x8, #8] : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.00 : ffff800010134ad0: b.hi ffff800010134bec <__update_load_avg_cfs_rq+0x184> // b.pmore : if (unlikely(local_n >= LOAD_AVG_PERIOD)) { 0.00 : ffff800010134ad4: cmp x12, #0x1f 0.00 : ffff800010134ad8: ldr w11, [x8, #24] 0.05 : ffff800010134adc: ldr x5, [x8, #16] 0.00 : ffff800010134ae0: b.hi ffff800010134bf8 <__update_load_avg_cfs_rq+0x190> // b.pmore : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.01 : ffff800010134ae4: adrp x6, ffff800010d06000 0.00 : ffff800010134ae8: add x6, x6, #0x450 : mul_u64_u32_shr(): : #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) : : #ifndef mul_u64_u32_shr : static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) : { : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff800010134aec: ldr w2, [x6, w12, uxtw #2] 0.00 : ffff800010134af0: mul x14, x2, x3 0.00 : ffff800010134af4: mul x6, x2, x5 0.01 : ffff800010134af8: umulh x3, x2, x3 0.03 : ffff800010134afc: umulh x5, x2, x5 0.00 : ffff800010134b00: extr x3, x3, x14, #32 0.00 : ffff800010134b04: extr x5, x5, x6, #32 0.02 : ffff800010134b08: stp x3, x5, [x8, #8] 0.00 : ffff800010134b0c: mul x2, x11, x2 0.00 : ffff800010134b10: lsr x2, x2, #32 : accumulate_sum(): : sa->util_sum = decay_load((u64)(sa->util_sum), periods); 0.00 : ffff800010134b14: str w2, [x8, #24] : delta %= 1024; 0.00 : ffff800010134b18: and x14, x4, #0x3ff 0.00 : ffff800010134b1c: mov w11, w14 : if (load) { 0.00 : ffff800010134b20: cbnz x9, ffff800010134bd0 <__update_load_avg_cfs_rq+0x168> : sa->period_contrib = delta; 0.01 : ffff800010134b24: str w14, [x8, #28] : if (runnable) 0.00 : ffff800010134b28: cbz x10, ffff800010134b3c <__update_load_avg_cfs_rq+0xd4> : sa->runnable_load_sum += runnable * contrib; 0.01 : ffff800010134b2c: ldr x2, [x8, #16] 0.00 : ffff800010134b30: mov w1, w0 0.00 : ffff800010134b34: madd x10, x1, x10, x2 0.01 : ffff800010134b38: str x10, [x8, #16] : if (running) 0.00 : ffff800010134b3c: cbz w13, ffff800010134b4c <__update_load_avg_cfs_rq+0xe4> : sa->util_sum += contrib << SCHED_CAPACITY_SHIFT; 0.00 : ffff800010134b40: ldr w1, [x8, #24] 0.00 : ffff800010134b44: add w0, w1, w0, lsl #10 0.01 : ffff800010134b48: str w0, [x8, #24] : ___update_load_sum(): : if (!accumulate_sum(delta, sa, load, runnable, running)) 0.03 : ffff800010134b4c: cbz w12, ffff800010134bc4 <__update_load_avg_cfs_rq+0x15c> : div_u64_rem(): : return dividend / divisor; 0.01 : ffff800010134b50: ldp x4, x3, [x8, #8] : ___update_load_avg(): : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010134b54: mov w0, #0xb67e // #46718 : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 0.13 : ffff800010134b58: ldr w2, [x8, #24] : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010134b5c: add w5, w11, w0 : __update_load_avg_cfs_rq(): : : ___update_load_avg(&cfs_rq->avg, 1, 1); : trace_pelt_cfs_tp(cfs_rq); : return 1; 0.00 : ffff800010134b60: mov w0, #0x1 // #1 : div_u64_rem(): 0.00 : ffff800010134b64: udiv x4, x4, x5 0.03 : ffff800010134b68: udiv x3, x3, x5 : ___update_load_avg(): : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 0.00 : ffff800010134b6c: udiv w1, w2, w5 : sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider); 0.00 : ffff800010134b70: stp x4, x3, [x8, #32] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010134b74: str x1, [x7, #176] : __update_load_avg_cfs_rq(): : } : : return 0; : } 0.02 : ffff800010134b78: ldp x29, x30, [sp], #16 0.00 : ffff800010134b7c: ret : return 0; 0.06 : ffff800010134b80: mov w0, #0x0 // #0 : } 0.00 : ffff800010134b84: ret : ___update_load_sum(): : sa->last_update_time = now; 0.00 : ffff800010134b88: str x0, [x1, #128] : __update_load_avg_cfs_rq(): : return 0; 0.00 : ffff800010134b8c: mov w0, #0x0 // #0 0.00 : ffff800010134b90: ret : if (___update_load_sum(now, &cfs_rq->avg, 0.01 : ffff800010134b94: cmp x2, #0x0 0.00 : ffff800010134b98: lsr x10, x10, #10 0.00 : ffff800010134b9c: cset w13, ne // ne = any : accumulate_sum(): : if (periods) { 0.00 : ffff800010134ba0: cbnz x12, ffff800010134ac8 <__update_load_avg_cfs_rq+0x60> : sa->period_contrib = delta; 0.00 : ffff800010134ba4: mov w11, w4 0.01 : ffff800010134ba8: str w4, [x8, #28] : sa->load_sum += load * contrib; 0.02 : ffff800010134bac: ldr x2, [x8, #8] 0.00 : ffff800010134bb0: mov w1, w0 0.00 : ffff800010134bb4: madd x9, x1, x9, x2 0.00 : ffff800010134bb8: str x9, [x8, #8] 0.00 : ffff800010134bbc: b ffff800010134b28 <__update_load_avg_cfs_rq+0xc0> : sa->period_contrib = delta; 4.16 : ffff800010134bc0: str w4, [x8, #28] : __update_load_avg_cfs_rq(): : return 0; 0.01 : ffff800010134bc4: mov w0, #0x0 // #0 : } 4.53 : ffff800010134bc8: ldp x29, x30, [sp], #16 0.00 : ffff800010134bcc: ret : accumulate_sum(): : contrib = __accumulate_pelt_segments(periods, 0.00 : ffff800010134bd0: mov w0, #0x400 // #1024 0.00 : ffff800010134bd4: mov w2, w14 0.00 : ffff800010134bd8: sub w1, w0, w1 0.00 : ffff800010134bdc: mov x0, x12 0.00 : ffff800010134be0: bl ffff8000101346b8 <__accumulate_pelt_segments> : sa->period_contrib = delta; 0.00 : ffff800010134be4: str w14, [x8, #28] 0.00 : ffff800010134be8: b ffff800010134bac <__update_load_avg_cfs_rq+0x144> : sa->runnable_load_sum = 0.00 : ffff800010134bec: mov w2, #0x0 // #0 0.00 : ffff800010134bf0: stp xzr, xzr, [x8, #8] 0.00 : ffff800010134bf4: b ffff800010134b14 <__update_load_avg_cfs_rq+0xac> : decay_load(): : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010134bf8: and x14, x12, #0x1f 0.00 : ffff800010134bfc: adrp x2, ffff800010d06000 0.00 : ffff800010134c00: add x2, x2, #0x450 : val >>= local_n / LOAD_AVG_PERIOD; 0.00 : ffff800010134c04: lsr w6, w12, #5 0.00 : ffff800010134c08: lsr x3, x3, x6 : mul_u64_u32_shr(): : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff800010134c0c: ldr w2, [x2, x14, lsl #2] : decay_load(): 0.00 : ffff800010134c10: lsr x5, x5, x6 0.00 : ffff800010134c14: lsr x11, x11, x6 : mul_u64_u32_shr(): 0.00 : ffff800010134c18: mul x14, x3, x2 0.00 : ffff800010134c1c: umulh x3, x3, x2 0.00 : ffff800010134c20: mul x6, x5, x2 0.00 : ffff800010134c24: umulh x5, x5, x2 0.00 : ffff800010134c28: extr x3, x3, x14, #32 0.00 : ffff800010134c2c: str x3, [x8, #8] 0.00 : ffff800010134c30: extr x3, x5, x6, #32 0.00 : ffff800010134c34: str x3, [x8, #16] 0.00 : ffff800010134c38: b ffff800010134b0c <__update_load_avg_cfs_rq+0xa4> Percent | Source code & Disassembly of vmlinux for cycles (16672 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010456448 : : bio_uninit(): : (*idx)++; : return bvl; : } : : void bio_uninit(struct bio *bio) : { 0.09 : ffff800010456448: stp x29, x30, [sp, #-32]! 0.00 : ffff80001045644c: mov x29, sp 2.47 : ffff800010456450: stp x19, x20, [sp, #16] 0.01 : ffff800010456454: mov x19, x0 : bio_disassociate_blkg(): : * : * Helper to disassociate the blkg from @bio if a blkg is associated. : */ : void bio_disassociate_blkg(struct bio *bio) : { : if (bio->bi_blkg) { 0.64 : ffff800010456458: ldr x20, [x0, #72] 0.00 : ffff80001045645c: cbz x20, ffff8000104564c0 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.49 : ffff800010456460: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.13 : ffff800010456464: ldr x0, [x20, #72] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.11 : ffff800010456468: tst x0, #0x3 0.31 : ffff80001045646c: b.ne ffff8000104564e4 // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 13.73 : ffff800010456470: mrs x1, sp_el0 : __read_once_size(): 2.23 : ffff800010456474: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010456478: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.57 : ffff80001045647c: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff800010456480: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.06 : ffff800010456484: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.06 : ffff800010456488: add x0, x0, x2 2.27 : ffff80001045648c: ldxr x5, [x0] 37.78 : ffff800010456490: add x5, x5, x3 0.05 : ffff800010456494: stxr w4, x5, [x0] 0.02 : ffff800010456498: cbnz w4, ffff80001045648c : __read_once_size(): : __READ_ONCE_SIZE; 1.47 : ffff80001045649c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.09 : ffff8000104564a0: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 3.48 : ffff8000104564a4: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000104564a8: cbz x0, ffff8000104564b4 : __read_once_size(): : __READ_ONCE_SIZE; 7.86 : ffff8000104564ac: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000104564b0: cbnz x0, ffff8000104564b8 : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff8000104564b4: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 11.23 : ffff8000104564b8: bl ffff800010160d20 <__rcu_read_unlock> : bio_disassociate_blkg(): : blkg_put(bio->bi_blkg); : bio->bi_blkg = NULL; 0.14 : ffff8000104564bc: str xzr, [x19, #72] : bio_integrity(): : : #if defined(CONFIG_BLK_DEV_INTEGRITY) : : static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) : { : if (bio->bi_opf & REQ_INTEGRITY) 3.23 : ffff8000104564c0: ldr w0, [x19, #16] 0.00 : ffff8000104564c4: tbz w0, #16, ffff8000104564d8 : bio_uninit(): : if (bio_integrity(bio)) 0.00 : ffff8000104564c8: ldr x0, [x19, #88] 0.00 : ffff8000104564cc: cbz x0, ffff8000104564d8 : bio_integrity_free(bio); 0.00 : ffff8000104564d0: mov x0, x19 0.00 : ffff8000104564d4: bl ffff8000104805e0 : } 5.56 : ffff8000104564d8: ldp x19, x20, [sp, #16] 4.93 : ffff8000104564dc: ldp x29, x30, [sp], #32 0.00 : ffff8000104564e0: ret : blkg_put(): : * blkg_put - put a blkg reference : * @blkg: blkg to put : */ : static inline void blkg_put(struct blkcg_gq *blkg) : { : percpu_ref_put(&blkg->refcnt); 0.00 : ffff8000104564e4: add x0, x20, #0x40 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000104564e8: b ffff800010456510 0.00 : ffff8000104564ec: b ffff800010456510 : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000104564f0: mov x1, #0x1 // #1 0.00 : ffff8000104564f4: neg x1, x1 0.00 : ffff8000104564f8: ldaddal x1, x2, [x0] 0.00 : ffff8000104564fc: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff800010456500: cbnz x1, ffff8000104564b8 : ref->release(ref); 0.00 : ffff800010456504: ldr x1, [x0, #16] 0.00 : ffff800010456508: blr x1 0.00 : ffff80001045650c: b ffff8000104564b8 : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff800010456510: mov x2, #0x1 // #1 0.00 : ffff800010456514: add x4, x20, #0x40 0.00 : ffff800010456518: b ffff800010458b08 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001045651c: cbnz x1, ffff8000104564b8 0.00 : ffff800010456520: b ffff800010456504 Percent | Source code & Disassembly of vmlinux for cycles (16441 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010717cf8 : : free_iova_fast(): : * This functions frees an iova range by trying to put it into the rcache, : * falling back to regular iova deallocation via free_iova() if this fails. : */ : void : free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) : { 0.18 : ffff800010717cf8: stp x29, x30, [sp, #-80]! : __order_base_2(): : ) : : static inline __attribute_const__ : int __order_base_2(unsigned long n) : { : return n > 1 ? ilog2(n - 1) + 1 : 0; 0.00 : ffff800010717cfc: cmp x2, #0x1 : free_iova_fast(): 0.00 : ffff800010717d00: mov x29, sp 0.02 : ffff800010717d04: stp x22, x23, [sp, #40] 0.00 : ffff800010717d08: mov x22, x0 0.00 : ffff800010717d0c: mov x23, x1 : __order_base_2(): 0.00 : ffff800010717d10: b.ls ffff800010717d48 // b.plast 0.00 : ffff800010717d14: sub x2, x2, #0x1 : __fls(): : * : * Undefined if no set bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __fls(unsigned long word) : { : return (sizeof(word) * 8) - 1 - __builtin_clzl(word); 0.00 : ffff800010717d18: mov x1, #0x3f // #63 0.00 : ffff800010717d1c: clz x2, x2 0.00 : ffff800010717d20: sub x1, x1, x2 : fls64(): : #elif BITS_PER_LONG == 64 : static __always_inline int fls64(__u64 x) : { : if (x == 0) : return 0; : return __fls(x) + 1; 0.00 : ffff800010717d24: add w1, w1, #0x1 : iova_rcache_insert(): : static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn, : unsigned long size) : { : unsigned int log_size = order_base_2(size); : : if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE) 0.00 : ffff800010717d28: cmp w1, #0x5 0.00 : ffff800010717d2c: b.ls ffff800010717f5c // b.plast : free_iova_fast(): : free_iova(iovad, pfn); 0.00 : ffff800010717d30: mov x1, x23 0.00 : ffff800010717d34: mov x0, x22 0.00 : ffff800010717d38: bl ffff800010717a60 : } 0.00 : ffff800010717d3c: ldp x22, x23, [sp, #40] 0.00 : ffff800010717d40: ldp x29, x30, [sp], #80 0.00 : ffff800010717d44: ret 0.37 : ffff800010717d48: stp x19, x20, [x29, #16] : iova_rcache_insert(): : unsigned int log_size = order_base_2(size); 0.00 : ffff800010717d4c: mov x1, #0x0 // #0 1.47 : ffff800010717d50: str x21, [x29, #32] : __order_base_2(): 0.00 : ffff800010717d54: mov x21, #0x80 // #128 1.40 : ffff800010717d58: stp x24, x25, [x29, #56] : __iova_rcache_insert(): : cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches); 0.00 : ffff800010717d5c: add x20, x1, x1, lsl #3 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.02 : ffff800010717d60: mrs x0, tpidr_el1 : __iova_rcache_insert(): 0.61 : ffff800010717d64: lsl x20, x20, #2 0.00 : ffff800010717d68: sub x20, x20, x1 0.00 : ffff800010717d6c: add x24, x22, x20, lsl #3 1.25 : ffff800010717d70: ldr x19, [x24, #400] 0.00 : ffff800010717d74: add x19, x19, x0 : spin_lock_irqsave(&cpu_rcache->lock, flags); 0.00 : ffff800010717d78: mov x0, x19 0.00 : ffff800010717d7c: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff800010717d80: mov x25, x0 : if (!iova_magazine_full(cpu_rcache->loaded)) { 31.28 : ffff800010717d84: ldr x2, [x19, #8] : iova_magazine_full(): : return (mag && mag->size == IOVA_MAG_SIZE); 0.00 : ffff800010717d88: cbz x2, ffff800010717dcc 25.56 : ffff800010717d8c: ldr x0, [x2] 0.00 : ffff800010717d90: cmp x0, #0x80 0.00 : ffff800010717d94: b.eq ffff800010717dfc // b.none : iova_magazine_push(): : mag->pfns[mag->size++] = pfn; 8.21 : ffff800010717d98: add x3, x2, x0, lsl #3 0.00 : ffff800010717d9c: add x0, x0, #0x1 1.69 : ffff800010717da0: str x0, [x2] : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff800010717da4: mov x1, x25 0.00 : ffff800010717da8: mov x0, x19 : iova_magazine_push(): 0.02 : ffff800010717dac: str x23, [x3, #8] : spin_unlock_irqrestore(): 0.00 : ffff800010717db0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.33 : ffff800010717db4: ldr x21, [x29, #32] 1.49 : ffff800010717db8: ldp x19, x20, [x29, #16] 0.01 : ffff800010717dbc: ldp x24, x25, [x29, #56] : free_iova_fast(): : } 24.82 : ffff800010717dc0: ldp x22, x23, [sp, #40] 0.05 : ffff800010717dc4: ldp x29, x30, [sp], #80 0.00 : ffff800010717dc8: ret 0.00 : ffff800010717dcc: ldr x3, [x2] : spin_unlock_irqrestore(): 0.00 : ffff800010717dd0: mov x1, x0 0.00 : ffff800010717dd4: mov x0, x19 : iova_magazine_push(): : mag->pfns[mag->size++] = pfn; 0.00 : ffff800010717dd8: add x4, x3, #0x1 0.00 : ffff800010717ddc: str x4, [x2] 0.00 : ffff800010717de0: lsl x3, x3, #3 0.00 : ffff800010717de4: str x23, [x3, #8] : spin_unlock_irqrestore(): 0.00 : ffff800010717de8: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010717dec: ldr x21, [x29, #32] 0.00 : ffff800010717df0: ldp x19, x20, [x29, #16] 0.00 : ffff800010717df4: ldp x24, x25, [x29, #56] 0.00 : ffff800010717df8: b ffff800010717dc0 0.11 : ffff800010717dfc: str x26, [x29, #72] : __iova_rcache_insert(): : } else if (!iova_magazine_full(cpu_rcache->prev)) { 0.23 : ffff800010717e00: ldr x3, [x19, #16] : iova_magazine_full(): : return (mag && mag->size == IOVA_MAG_SIZE); 0.00 : ffff800010717e04: cbz x3, ffff800010717e60 0.01 : ffff800010717e08: ldr x0, [x3] 0.00 : ffff800010717e0c: cmp x0, #0x80 0.00 : ffff800010717e10: b.eq ffff800010717e80 // b.none : __iova_rcache_insert(): : swap(cpu_rcache->prev, cpu_rcache->loaded); 0.00 : ffff800010717e14: stp x3, x2, [x19, #8] 0.00 : ffff800010717e18: ldr x0, [x3] : iova_magazine_full(): : return (mag && mag->size == IOVA_MAG_SIZE); 0.00 : ffff800010717e1c: cmp x0, #0x80 0.00 : ffff800010717e20: b.eq ffff800010717e5c // b.none : iova_magazine_push(): : mag->pfns[mag->size++] = pfn; 0.00 : ffff800010717e24: add x2, x3, x0, lsl #3 0.00 : ffff800010717e28: add x0, x0, #0x1 : spin_unlock_irqrestore(): 0.00 : ffff800010717e2c: mov x1, x25 : iova_magazine_push(): 0.00 : ffff800010717e30: str x0, [x3] : spin_unlock_irqrestore(): 0.00 : ffff800010717e34: mov x0, x19 : iova_magazine_push(): 0.00 : ffff800010717e38: str x23, [x2, #8] : spin_unlock_irqrestore(): 0.00 : ffff800010717e3c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010717e40: ldp x19, x20, [x29, #16] 0.00 : ffff800010717e44: ldr x21, [x29, #32] 0.00 : ffff800010717e48: ldp x24, x25, [x29, #56] 0.23 : ffff800010717e4c: ldr x26, [x29, #72] : free_iova_fast(): : } 0.00 : ffff800010717e50: ldp x22, x23, [sp, #40] 0.00 : ffff800010717e54: ldp x29, x30, [sp], #80 0.00 : ffff800010717e58: ret : iova_magazine_push(): : BUG_ON(iova_magazine_full(mag)); 0.00 : ffff800010717e5c: brk #0x800 : __iova_rcache_insert(): : swap(cpu_rcache->prev, cpu_rcache->loaded); 0.00 : ffff800010717e60: stp xzr, x2, [x19, #8] : spin_unlock_irqrestore(): 0.00 : ffff800010717e64: mov x1, x25 0.00 : ffff800010717e68: mov x0, x19 : iova_magazine_push(): : mag->pfns[mag->size++] = pfn; 0.00 : ffff800010717e6c: ldr x2, [x3] 0.00 : ffff800010717e70: add x4, x2, #0x1 0.00 : ffff800010717e74: str x4, [x3] 0.00 : ffff800010717e78: lsl x2, x2, #3 0.00 : ffff800010717e7c: b ffff800010717e38 : kmalloc(): : index = kmalloc_index(size); : : if (!index) : return ZERO_SIZE_PTR; : : return kmem_cache_alloc_trace( 0.13 : ffff800010717e80: adrp x0, ffff8000112f2000 : kmem_cache_alloc_trace(): : void *ret = kmem_cache_alloc(s, flags); 0.00 : ffff800010717e84: mov w1, #0xb20 // #2848 0.02 : ffff800010717e88: ldr x0, [x0, #2456] 0.00 : ffff800010717e8c: bl ffff800010252238 0.01 : ffff800010717e90: mov x26, x0 : __iova_rcache_insert(): : if (new_mag) { 0.00 : ffff800010717e94: cbz x0, ffff800010717f3c : iova_rcache_insert(): : return false; : : return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn); 0.01 : ffff800010717e98: add x21, x22, x21 : spin_lock(): : raw_spin_lock(&lock->rlock); 0.00 : ffff800010717e9c: mov x0, x21 0.00 : ffff800010717ea0: bl ffff800010cd80f0 <_raw_spin_lock> : __iova_rcache_insert(): : if (rcache->depot_size < MAX_GLOBAL_MAGS) { 0.07 : ffff800010717ea4: ldr x0, [x24, #136] 0.00 : ffff800010717ea8: cmp x0, #0x1f 0.00 : ffff800010717eac: b.ls ffff800010717f1c // b.plast : mag_to_free = cpu_rcache->loaded; 0.00 : ffff800010717eb0: ldr x20, [x19, #8] : spin_unlock(): : raw_spin_unlock(&lock->rlock); 0.07 : ffff800010717eb4: mov x0, x21 0.00 : ffff800010717eb8: bl ffff800010cd7c38 <_raw_spin_unlock> : __iova_rcache_insert(): : cpu_rcache->loaded = new_mag; 0.00 : ffff800010717ebc: str x26, [x19, #8] 0.00 : ffff800010717ec0: ldr x0, [x26] : iova_magazine_full(): : return (mag && mag->size == IOVA_MAG_SIZE); 0.00 : ffff800010717ec4: cmp x0, #0x80 0.00 : ffff800010717ec8: b.eq ffff800010717e5c // b.none : iova_magazine_push(): : mag->pfns[mag->size++] = pfn; 0.00 : ffff800010717ecc: add x2, x26, x0, lsl #3 0.00 : ffff800010717ed0: add x0, x0, #0x1 0.01 : ffff800010717ed4: str x0, [x26] : spin_unlock_irqrestore(): : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff800010717ed8: mov x1, x25 0.00 : ffff800010717edc: mov x0, x19 : iova_magazine_push(): 0.00 : ffff800010717ee0: str x23, [x2, #8] : spin_unlock_irqrestore(): 0.00 : ffff800010717ee4: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : __iova_rcache_insert(): : if (mag_to_free) { 0.00 : ffff800010717ee8: cbz x20, ffff800010717e40 : iova_magazine_free_pfns(mag_to_free, iovad); 0.00 : ffff800010717eec: mov x1, x22 0.00 : ffff800010717ef0: mov x0, x20 0.00 : ffff800010717ef4: bl ffff800010717a90 : iova_magazine_free(): : kfree(mag); 0.00 : ffff800010717ef8: mov x0, x20 0.00 : ffff800010717efc: bl ffff800010253890 0.00 : ffff800010717f00: ldr x21, [x29, #32] 0.00 : ffff800010717f04: ldp x19, x20, [x29, #16] 0.00 : ffff800010717f08: ldp x24, x25, [x29, #56] 0.00 : ffff800010717f0c: ldr x26, [x29, #72] : free_iova_fast(): : } 0.00 : ffff800010717f10: ldp x22, x23, [sp, #40] 0.00 : ffff800010717f14: ldp x29, x30, [sp], #80 0.00 : ffff800010717f18: ret : __iova_rcache_insert(): : rcache->depot[rcache->depot_size++] = 0.30 : ffff800010717f1c: add x1, x20, x0 0.00 : ffff800010717f20: add x0, x0, #0x1 0.00 : ffff800010717f24: add x1, x1, #0x12 0.00 : ffff800010717f28: str x0, [x24, #136] : struct iova_magazine *mag_to_free = NULL; 0.00 : ffff800010717f2c: mov x20, #0x0 // #0 : cpu_rcache->loaded; 0.00 : ffff800010717f30: ldr x0, [x19, #8] : rcache->depot[rcache->depot_size++] = 0.00 : ffff800010717f34: str x0, [x22, x1, lsl #3] 0.00 : ffff800010717f38: b ffff800010717eb4 : spin_unlock_irqrestore(): 0.00 : ffff800010717f3c: mov x1, x25 0.00 : ffff800010717f40: mov x0, x19 0.00 : ffff800010717f44: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010717f48: ldr x21, [x29, #32] 0.00 : ffff800010717f4c: ldp x19, x20, [x29, #16] 0.00 : ffff800010717f50: ldp x24, x25, [x29, #56] 0.00 : ffff800010717f54: ldr x26, [x29, #72] 0.00 : ffff800010717f58: b ffff800010717d30 0.00 : ffff800010717f5c: mov x0, #0x80 // #128 0.00 : ffff800010717f60: str x21, [x29, #32] 0.00 : ffff800010717f64: mov w21, #0x118 // #280 0.00 : ffff800010717f68: stp x19, x20, [x29, #16] 0.00 : ffff800010717f6c: stp x24, x25, [x29, #56] 0.00 : ffff800010717f70: umaddl x21, w1, w21, x0 0.00 : ffff800010717f74: b ffff800010717d5c Percent | Source code & Disassembly of vmlinux for cycles (16345 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001012f7a8 <__wake_up_common>: : __wake_up_common(): : * zero in this (rare) case, and we handle it by continuing to scan the queue. : */ : static int __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode, : int nr_exclusive, int wake_flags, void *key, : wait_queue_entry_t *bookmark) : { 0.00 : ffff80001012f7a8: stp x29, x30, [sp, #-96]! 0.00 : ffff80001012f7ac: mov x29, sp 0.01 : ffff80001012f7b0: str x22, [sp, #40] 0.00 : ffff80001012f7b4: str x26, [sp, #72] : wait_queue_entry_t *curr, *next; : int cnt = 0; : : lockdep_assert_held(&wq_head->lock); : : if (bookmark && (bookmark->flags & WQ_FLAG_BOOKMARK)) { 0.00 : ffff80001012f7b8: cbz x5, ffff80001012f7c4 <__wake_up_common+0x1c> 1.39 : ffff80001012f7bc: ldr w6, [x5] 0.00 : ffff80001012f7c0: tbnz w6, #2, ffff80001012f8a8 <__wake_up_common+0x100> : curr = list_next_entry(bookmark, entry); : : list_del(&bookmark->entry); : bookmark->flags = 0; : } else : curr = list_first_entry(&wq_head->head, wait_queue_entry_t, entry); 14.70 : ffff80001012f7c4: ldr x6, [x0, #8] 0.00 : ffff80001012f7c8: sub x6, x6, #0x18 : : if (&curr->entry == &wq_head->head) 0.00 : ffff80001012f7cc: add x22, x0, #0x8 0.00 : ffff80001012f7d0: add x0, x6, #0x18 : return nr_exclusive; 32.17 : ffff80001012f7d4: mov w26, w2 : if (&curr->entry == &wq_head->head) 0.00 : ffff80001012f7d8: cmp x22, x0 0.00 : ffff80001012f7dc: b.eq ffff80001012f878 <__wake_up_common+0xd0> // b.none 0.00 : ffff80001012f7e0: stp x19, x20, [x29, #16] 3.38 : ffff80001012f7e4: str x21, [x29, #32] 0.00 : ffff80001012f7e8: mov x21, x5 0.00 : ffff80001012f7ec: stp x23, x24, [x29, #48] 0.00 : ffff80001012f7f0: mov w24, w3 0.04 : ffff80001012f7f4: str x25, [x29, #64] 0.00 : ffff80001012f7f8: mov w23, w1 0.09 : ffff80001012f7fc: str x27, [x29, #80] 0.00 : ffff80001012f800: mov x25, x4 : int cnt = 0; 0.00 : ffff80001012f804: mov w27, #0x0 // #0 : : list_for_each_entry_safe_from(curr, next, &wq_head->head, entry) { 1.53 : ffff80001012f808: ldr x19, [x6, #24] 0.00 : ffff80001012f80c: sub x19, x19, #0x18 : unsigned flags = curr->flags; 28.53 : ffff80001012f810: ldr w20, [x6] : int ret; : : if (flags & WQ_FLAG_BOOKMARK) : continue; : : ret = curr->func(curr, mode, wake_flags, key); 0.00 : ffff80001012f814: mov x0, x6 0.00 : ffff80001012f818: mov x3, x25 0.00 : ffff80001012f81c: mov w2, w24 2.80 : ffff80001012f820: mov w1, w23 : if (flags & WQ_FLAG_BOOKMARK) 0.00 : ffff80001012f824: tbnz w20, #2, ffff80001012f88c <__wake_up_common+0xe4> : ret = curr->func(curr, mode, wake_flags, key); 0.01 : ffff80001012f828: ldr x4, [x6, #16] 0.00 : ffff80001012f82c: blr x4 : if (ret < 0) 0.17 : ffff80001012f830: cmp w0, #0x0 0.00 : ffff80001012f834: b.lt ffff80001012f864 <__wake_up_common+0xbc> // b.tstop : break; : if (ret && (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) 0.00 : ffff80001012f838: b.eq ffff80001012f848 <__wake_up_common+0xa0> // b.none 0.01 : ffff80001012f83c: tbz w20, #0, ffff80001012f848 <__wake_up_common+0xa0> 0.00 : ffff80001012f840: subs w26, w26, #0x1 0.00 : ffff80001012f844: b.eq ffff80001012f864 <__wake_up_common+0xbc> // b.none : break; : : if (bookmark && (++cnt > WAITQUEUE_WALK_BREAK_CNT) && 0.36 : ffff80001012f848: cbz x21, ffff80001012f88c <__wake_up_common+0xe4> 0.02 : ffff80001012f84c: add w27, w27, #0x1 0.00 : ffff80001012f850: cmp w27, #0x40 0.00 : ffff80001012f854: b.le ffff80001012f88c <__wake_up_common+0xe4> : (&next->entry != &wq_head->head)) { 0.00 : ffff80001012f858: add x0, x19, #0x18 : if (bookmark && (++cnt > WAITQUEUE_WALK_BREAK_CNT) && 0.00 : ffff80001012f85c: cmp x22, x0 0.00 : ffff80001012f860: b.ne ffff80001012f8d4 <__wake_up_common+0x12c> // b.any 0.04 : ffff80001012f864: ldp x19, x20, [x29, #16] 0.82 : ffff80001012f868: ldr x21, [x29, #32] 0.07 : ffff80001012f86c: ldp x23, x24, [x29, #48] 0.25 : ffff80001012f870: ldr x25, [x29, #64] 0.51 : ffff80001012f874: ldr x27, [x29, #80] : break; : } : } : : return nr_exclusive; : } 0.02 : ffff80001012f878: mov w0, w26 0.12 : ffff80001012f87c: ldr x22, [sp, #40] 0.31 : ffff80001012f880: ldr x26, [sp, #72] 0.77 : ffff80001012f884: ldp x29, x30, [sp], #96 0.00 : ffff80001012f888: ret : list_for_each_entry_safe_from(curr, next, &wq_head->head, entry) { 0.02 : ffff80001012f88c: mov x7, x19 0.00 : ffff80001012f890: mov x6, x19 7.33 : ffff80001012f894: ldr x19, [x7, #24]! 0.00 : ffff80001012f898: sub x19, x19, #0x18 0.00 : ffff80001012f89c: cmp x22, x7 0.00 : ffff80001012f8a0: b.ne ffff80001012f810 <__wake_up_common+0x68> // b.any 4.53 : ffff80001012f8a4: b ffff80001012f864 <__wake_up_common+0xbc> : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff80001012f8a8: ldp x7, x8, [x5, #24] : __list_del(): : next->prev = prev; 0.00 : ffff80001012f8ac: str x8, [x7, #8] : list_del(): : * in an undefined state. : */ : static inline void list_del(struct list_head *entry) : { : __list_del_entry(entry); : entry->next = LIST_POISON1; 0.00 : ffff80001012f8b0: mov x10, #0x100 // #256 : entry->prev = LIST_POISON2; 0.00 : ffff80001012f8b4: mov x9, #0x122 // #290 : entry->next = LIST_POISON1; 0.00 : ffff80001012f8b8: movk x10, #0xdead, lsl #48 : entry->prev = LIST_POISON2; 0.00 : ffff80001012f8bc: movk x9, #0xdead, lsl #48 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012f8c0: str x7, [x8] : __wake_up_common(): : curr = list_next_entry(bookmark, entry); 0.00 : ffff80001012f8c4: sub x6, x7, #0x18 : bookmark->flags = 0; 0.00 : ffff80001012f8c8: str wzr, [x5] : list_del(): 0.00 : ffff80001012f8cc: stp x10, x9, [x5, #24] : __wake_up_common(): 0.00 : ffff80001012f8d0: b ffff80001012f7cc <__wake_up_common+0x24> : bookmark->flags = WQ_FLAG_BOOKMARK; 0.00 : ffff80001012f8d4: mov x1, x21 0.00 : ffff80001012f8d8: mov w2, #0x4 // #4 0.00 : ffff80001012f8dc: str w2, [x1], #24 : list_add_tail(): : __list_add(new, head->prev, head); 0.00 : ffff80001012f8e0: ldr x2, [x19, #32] : __list_add(): : next->prev = new; 0.00 : ffff80001012f8e4: str x1, [x19, #32] : new->prev = prev; 0.00 : ffff80001012f8e8: stp x0, x2, [x21, #24] : __write_once_size(): 0.00 : ffff80001012f8ec: str x1, [x2] 0.00 : ffff80001012f8f0: b ffff80001012f864 <__wake_up_common+0xbc> Percent | Source code & Disassembly of vmlinux for cycles (14610 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff8000102c6c58 : : blkdev_bio_end_io(): : : return blk_poll(q, READ_ONCE(kiocb->ki_cookie), wait); : } : : static void blkdev_bio_end_io(struct bio *bio) : { 2.59 : ffff8000102c6c58: stp x29, x30, [sp, #-48]! 0.00 : ffff8000102c6c5c: mov x29, sp 0.87 : ffff8000102c6c60: stp x19, x20, [sp, #16] 0.00 : ffff8000102c6c64: mov x20, x0 0.47 : ffff8000102c6c68: str x22, [sp, #40] : struct blkdev_dio *dio = bio->bi_private; 0.50 : ffff8000102c6c6c: ldr x19, [x0, #64] : bool should_dirty = dio->should_dirty; : : if (bio->bi_status && !dio->bio.bi_status) 2.45 : ffff8000102c6c70: ldrb w1, [x0, #26] : bool should_dirty = dio->should_dirty; 0.33 : ffff8000102c6c74: ldrb w0, [x19, #20] 0.01 : ffff8000102c6c78: ubfx x22, x0, #1, #1 : if (bio->bi_status && !dio->bio.bi_status) 0.00 : ffff8000102c6c7c: cbz w1, ffff8000102c6c88 0.00 : ffff8000102c6c80: ldrb w2, [x19, #50] 0.00 : ffff8000102c6c84: cbz w2, ffff8000102c6d30 : dio->bio.bi_status = bio->bi_status; : : if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) { 3.36 : ffff8000102c6c88: tbnz w0, #0, ffff8000102c6d38 0.27 : ffff8000102c6c8c: str x21, [x29, #32] 2.40 : ffff8000102c6c90: ldr x21, [x19] : if (!dio->is_sync) { 0.00 : ffff8000102c6c94: tbnz w0, #2, ffff8000102c6cf8 : struct kiocb *iocb = dio->iocb; : ssize_t ret; : : if (likely(!dio->bio.bi_status)) { 0.06 : ffff8000102c6c98: ldrb w0, [x19, #50] 0.00 : ffff8000102c6c9c: cbnz w0, ffff8000102c6d94 : ret = dio->size; 0.19 : ffff8000102c6ca0: ldr x1, [x19, #8] : iocb->ki_pos += ret; 1.44 : ffff8000102c6ca4: ldr x2, [x21, #8] 0.03 : ffff8000102c6ca8: add x2, x2, x1 77.92 : ffff8000102c6cac: str x2, [x21, #8] : } else { : ret = blk_status_to_errno(dio->bio.bi_status); : } : : dio->iocb->ki_complete(iocb, ret, 0); 1.28 : ffff8000102c6cb0: ldr x3, [x19] 0.00 : ffff8000102c6cb4: mov x0, x21 0.00 : ffff8000102c6cb8: mov x2, #0x0 // #0 0.21 : ffff8000102c6cbc: ldr x3, [x3, #16] 0.09 : ffff8000102c6cc0: blr x3 : if (dio->multi_bio) 0.24 : ffff8000102c6cc4: ldrb w0, [x19, #20] 0.02 : ffff8000102c6cc8: tbnz w0, #0, ffff8000102c6d84 2.68 : ffff8000102c6ccc: ldr x21, [x29, #32] : WRITE_ONCE(dio->waiter, NULL); : blk_wake_io_task(waiter); : } : } : : if (should_dirty) { 0.00 : ffff8000102c6cd0: cbnz w22, ffff8000102c6d18 : bio_check_pages_dirty(bio); : } else { : bio_release_pages(bio, false); 0.00 : ffff8000102c6cd4: mov w1, #0x0 // #0 0.00 : ffff8000102c6cd8: mov x0, x20 0.00 : ffff8000102c6cdc: bl ffff800010457518 : bio_put(bio); 0.00 : ffff8000102c6ce0: mov x0, x20 0.00 : ffff8000102c6ce4: bl ffff800010456ab8 : } : } 0.00 : ffff8000102c6ce8: ldp x19, x20, [sp, #16] 0.00 : ffff8000102c6cec: ldr x22, [sp, #40] 0.00 : ffff8000102c6cf0: ldp x29, x30, [sp], #48 0.00 : ffff8000102c6cf4: ret : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000102c6cf8: str xzr, [x19] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000102c6cfc: mrs x0, sp_el0 : blk_wake_io_task(): : /* : * If we're polling, the task itself is doing the completions. For : * that case, we don't need to signal a wakeup, it's enough to just : * mark us as RUNNING. : */ : if (waiter == current) 0.00 : ffff8000102c6d00: cmp x21, x0 0.00 : ffff8000102c6d04: b.eq ffff8000102c6d78 // b.none : __set_current_state(TASK_RUNNING); : else : wake_up_process(waiter); 0.00 : ffff8000102c6d08: mov x0, x21 0.00 : ffff8000102c6d0c: bl ffff800010116f00 0.00 : ffff8000102c6d10: ldr x21, [x29, #32] : blkdev_bio_end_io(): : if (should_dirty) { 0.00 : ffff8000102c6d14: cbz w22, ffff8000102c6cd4 : bio_check_pages_dirty(bio); 0.32 : ffff8000102c6d18: mov x0, x20 0.00 : ffff8000102c6d1c: bl ffff800010458508 : } 0.10 : ffff8000102c6d20: ldp x19, x20, [sp, #16] 2.19 : ffff8000102c6d24: ldr x22, [sp, #40] 0.00 : ffff8000102c6d28: ldp x29, x30, [sp], #48 0.00 : ffff8000102c6d2c: ret : dio->bio.bi_status = bio->bi_status; 0.00 : ffff8000102c6d30: strb w1, [x19, #50] : if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) { 0.00 : ffff8000102c6d34: tbz w0, #0, ffff8000102c6c8c 0.00 : ffff8000102c6d38: add x1, x19, #0x10 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102c6d3c: b ffff8000102c6d64 0.00 : ffff8000102c6d40: b ffff8000102c6d64 : __lse_atomic_sub_return(): : } : : ATOMIC_OP_SUB_RETURN(_relaxed, ) : ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC_OP_SUB_RETURN(_release, l, "memory") : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000102c6d44: mov w0, #0x1 // #1 0.00 : ffff8000102c6d48: neg w0, w0 0.00 : ffff8000102c6d4c: ldaddal w0, w2, [x1] 0.00 : ffff8000102c6d50: add w0, w0, w2 : blkdev_bio_end_io(): 0.00 : ffff8000102c6d54: cbnz w0, ffff8000102c6cd0 0.00 : ffff8000102c6d58: str x21, [x29, #32] 0.00 : ffff8000102c6d5c: ldrb w0, [x19, #20] 0.00 : ffff8000102c6d60: b ffff8000102c6c90 : __ll_sc_atomic_sub_return(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000102c6d64: mov w1, #0x1 // #1 0.00 : ffff8000102c6d68: add x3, x19, #0x10 0.00 : ffff8000102c6d6c: b ffff8000102c830c : blkdev_bio_end_io(): 0.00 : ffff8000102c6d70: cbnz w0, ffff8000102c6cd0 0.00 : ffff8000102c6d74: b ffff8000102c6d58 : blk_wake_io_task(): : __set_current_state(TASK_RUNNING); 0.00 : ffff8000102c6d78: str xzr, [x21, #24] 0.00 : ffff8000102c6d7c: ldr x21, [x29, #32] 0.00 : ffff8000102c6d80: b ffff8000102c6cd0 : blkdev_bio_end_io(): : bio_put(&dio->bio); 0.00 : ffff8000102c6d84: add x0, x19, #0x18 0.00 : ffff8000102c6d88: bl ffff800010456ab8 0.00 : ffff8000102c6d8c: ldr x21, [x29, #32] 0.00 : ffff8000102c6d90: b ffff8000102c6cd0 : ret = blk_status_to_errno(dio->bio.bi_status); 0.00 : ffff8000102c6d94: bl ffff800010459fd0 0.00 : ffff8000102c6d98: sxtw x1, w0 0.00 : ffff8000102c6d9c: b ffff8000102c6cb0 Percent | Source code & Disassembly of vmlinux for cycles (13696 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010160d20 <__rcu_read_unlock>: : __rcu_read_unlock(): : * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then : * invoke rcu_read_unlock_special() to clean up after a context switch : * in an RCU read-side critical section and other special cases. : */ : void __rcu_read_unlock(void) : { 10.46 : ffff800010160d20: stp x29, x30, [sp, #-32]! 0.01 : ffff800010160d24: mov x29, sp 8.46 : ffff800010160d28: str x19, [sp, #16] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 2.55 : ffff800010160d2c: mrs x19, sp_el0 : __rcu_read_unlock(): : struct task_struct *t = current; : : if (rcu_preempt_depth() != 1) { 4.61 : ffff800010160d30: ldr w0, [x19, #776] 2.26 : ffff800010160d34: cmp w0, #0x1 0.00 : ffff800010160d38: b.eq ffff800010160d50 <__rcu_read_unlock+0x30> // b.none : rcu_preempt_read_exit(): : current->rcu_read_lock_nesting--; 2.46 : ffff800010160d3c: sub w0, w0, #0x1 0.92 : ffff800010160d40: str w0, [x19, #776] : __rcu_read_unlock(): : if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { : int rrln = rcu_preempt_depth(); : : WARN_ON_ONCE(rrln < 0 && rrln > RCU_NEST_NMAX); : } : } 7.94 : ffff800010160d44: ldr x19, [sp, #16] 9.03 : ffff800010160d48: ldp x29, x30, [sp], #32 0.00 : ffff800010160d4c: ret : rcu_preempt_depth_set(): : current->rcu_read_lock_nesting = val; 19.35 : ffff800010160d50: mov w1, #0x80000001 // #-2147483647 7.34 : ffff800010160d54: str w1, [x19, #776] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.10 : ffff800010160d58: ldr w1, [x19, #780] : __rcu_read_unlock(): : if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s))) 0.00 : ffff800010160d5c: cbnz w1, ffff800010160d74 <__rcu_read_unlock+0x54> : get_current(): 4.96 : ffff800010160d60: mrs x0, sp_el0 : rcu_preempt_depth_set(): : current->rcu_read_lock_nesting = val; 8.45 : ffff800010160d64: str wzr, [x0, #776] : __rcu_read_unlock(): : } 6.38 : ffff800010160d68: ldr x19, [sp, #16] 3.56 : ffff800010160d6c: ldp x29, x30, [sp], #32 0.12 : ffff800010160d70: ret : __read_once_size(): 0.00 : ffff800010160d74: ldr w1, [x19, #16] 0.01 : ffff800010160d78: ldr w2, [x19, #16] : rcu_read_unlock_special(): : bool preempt_bh_were_disabled = : !!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)); : bool irqs_were_disabled; : : /* NMI handlers cannot block and cannot safely manipulate state. */ : if (in_nmi()) 0.00 : ffff800010160d7c: tbnz w2, #20, ffff800010160d60 <__rcu_read_unlock+0x40> 0.00 : ffff800010160d80: str x20, [x29, #24] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.02 : ffff800010160d84: mrs x20, daif : arch_irqs_disabled_flags(): : asm volatile(ALTERNATIVE( : "and %w0, %w1, #" __stringify(PSR_I_BIT), : "eor %w0, %w1, #" __stringify(GIC_PRIO_IRQON), : ARM64_HAS_IRQ_PRIO_MASKING) : : "=&r" (res) : : "r" ((int) flags) 0.00 : ffff800010160d88: mov w2, w20 : asm volatile(ALTERNATIVE( 0.00 : ffff800010160d8c: and w3, w20, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff800010160d90: cbnz w3, ffff800010160d9c <__rcu_read_unlock+0x7c> : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160d94: mov x3, #0x60 // #96 0.00 : ffff800010160d98: msr daifset, #0x2 : arch_irqs_disabled_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160d9c: and w3, w2, #0x80 : rcu_read_unlock_special(): : return; : : local_irq_save(flags); : irqs_were_disabled = irqs_disabled_flags(flags); : if (preempt_bh_were_disabled || irqs_were_disabled) { 0.00 : ffff800010160da0: and w1, w1, #0xffff 0.00 : ffff800010160da4: orr w1, w1, w3 0.00 : ffff800010160da8: cbz w1, ffff800010160e88 <__rcu_read_unlock+0x168> : bool exp; : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); : struct rcu_node *rnp = rdp->mynode; : : exp = (t->rcu_blocked_node && t->rcu_blocked_node->exp_tasks) || 0.00 : ffff800010160dac: ldr x2, [x19, #800] : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff800010160db0: adrp x1, ffff800011529000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010160db4: mrs x4, tpidr_el1 : rcu_read_unlock_special(): 0.00 : ffff800010160db8: add x1, x1, #0xac0 0.00 : ffff800010160dbc: add x1, x1, x4 : (rdp->grpmask & READ_ONCE(rnp->expmask)) || 0.00 : ffff800010160dc0: cbz x2, ffff800010160e18 <__rcu_read_unlock+0xf8> : exp = (t->rcu_blocked_node && t->rcu_blocked_node->exp_tasks) || 0.00 : ffff800010160dc4: ldr x2, [x2, #152] 0.00 : ffff800010160dc8: cbz x2, ffff800010160e18 <__rcu_read_unlock+0xf8> : tick_nohz_full_cpu(rdp->cpu); : // Need to defer quiescent state until everything is enabled. : if (irqs_were_disabled && use_softirq && 0.00 : ffff800010160dcc: cbz w3, ffff800010160e2c <__rcu_read_unlock+0x10c> 0.00 : ffff800010160dd0: adrp x2, ffff800011922000 0.00 : ffff800010160dd4: ldrb w2, [x2, #2864] 0.00 : ffff800010160dd8: cbz w2, ffff800010160e2c <__rcu_read_unlock+0x10c> : get_current(): 0.00 : ffff800010160ddc: mrs x2, sp_el0 : __read_once_size(): 0.00 : ffff800010160de0: ldr w2, [x2, #16] : rcu_read_unlock_special(): 0.00 : ffff800010160de4: tst w2, #0x1fff00 0.00 : ffff800010160de8: b.ne ffff800010160df8 <__rcu_read_unlock+0xd8> // b.any : (in_interrupt() || 0.00 : ffff800010160dec: cbz w0, ffff800010160e2c <__rcu_read_unlock+0x10c> : (exp && !t->rcu_read_unlock_special.b.deferred_qs))) { 0.00 : ffff800010160df0: ldrb w2, [x19, #783] 0.00 : ffff800010160df4: cbnz w2, ffff800010160e2c <__rcu_read_unlock+0x10c> : // Using softirq, safe to awaken, and we get : // no help from enabling irqs, unlike bh/preempt. : raise_softirq_irqoff(RCU_SOFTIRQ); 0.00 : ffff800010160df8: mov w0, #0x9 // #9 0.00 : ffff800010160dfc: bl ffff8000100ecd98 : rcu_preempt_deferred_qs_handler); : rdp->defer_qs_iw_pending = true; : irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); : } : } : t->rcu_read_unlock_special.b.deferred_qs = true; 0.00 : ffff800010160e00: mov w0, #0x1 // #1 0.00 : ffff800010160e04: strb w0, [x19, #783] : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff800010160e08: msr daif, x20 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010160e0c: nop 0.00 : ffff800010160e10: ldr x20, [x29, #24] 0.00 : ffff800010160e14: b ffff800010160d60 <__rcu_read_unlock+0x40> : rcu_read_unlock_special(): : (rdp->grpmask & READ_ONCE(rnp->expmask)) || 0.00 : ffff800010160e18: ldp x2, x0, [x1, #24] : __read_once_size(): 0.00 : ffff800010160e1c: ldr x2, [x2, #64] : rcu_read_unlock_special(): : exp = (t->rcu_blocked_node && t->rcu_blocked_node->exp_tasks) || 0.00 : ffff800010160e20: tst x2, x0 0.00 : ffff800010160e24: cset w0, ne // ne = any 0.00 : ffff800010160e28: b ffff800010160dcc <__rcu_read_unlock+0xac> : get_current(): 0.00 : ffff800010160e2c: mrs x2, sp_el0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010160e30: b ffff800010160e80 <__rcu_read_unlock+0x160> 0.00 : ffff800010160e34: b ffff800010160e80 <__rcu_read_unlock+0x160> : __lse_atomic64_or(): : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) 0.00 : ffff800010160e38: mov x4, #0x2 // #2 0.00 : ffff800010160e3c: stset x4, [x2] : get_current(): 0.00 : ffff800010160e40: mrs x2, sp_el0 : set_preempt_need_resched(): : task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \ : } while (0) : : static inline void set_preempt_need_resched(void) : { : current_thread_info()->preempt.need_resched = 0; 0.00 : ffff800010160e44: str wzr, [x2, #20] : rcu_read_unlock_special(): : if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled && 0.00 : ffff800010160e48: cbz w3, ffff800010160e00 <__rcu_read_unlock+0xe0> 0.00 : ffff800010160e4c: ldrb w2, [x1, #72] : !rdp->defer_qs_iw_pending && exp) { 0.00 : ffff800010160e50: bic x0, x0, x2 0.00 : ffff800010160e54: tbz w0, #0, ffff800010160e00 <__rcu_read_unlock+0xe0> : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010160e58: mov x0, x1 : init_irq_work(): : : static inline : void init_irq_work(struct irq_work *work, void (*func)(struct irq_work *)) : { : atomic_set(&work->flags, 0); : work->func = func; 0.00 : ffff800010160e5c: adrp x2, ffff80001015c000 0.00 : ffff800010160e60: add x2, x2, #0x1c8 : rcu_read_unlock_special(): : rdp->defer_qs_iw_pending = true; 0.00 : ffff800010160e64: mov w3, #0x1 // #1 : __write_once_size(): 0.00 : ffff800010160e68: str wzr, [x0, #48]! : init_irq_work(): 0.00 : ffff800010160e6c: str x2, [x0, #16] : rcu_read_unlock_special(): 0.00 : ffff800010160e70: strb w3, [x1, #72] : irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); 0.00 : ffff800010160e74: ldr w1, [x1, #352] 0.00 : ffff800010160e78: bl ffff8000101b4700 0.00 : ffff800010160e7c: b ffff800010160e00 <__rcu_read_unlock+0xe0> : __ll_sc_atomic64_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010160e80: b ffff800010162624 0.00 : ffff800010160e84: b ffff800010160e40 <__rcu_read_unlock+0x120> : rcu_read_unlock_special(): : local_irq_restore(flags); : return; : } : rcu_preempt_deferred_qs_irqrestore(t, flags); 0.01 : ffff800010160e88: mov x1, x20 0.00 : ffff800010160e8c: mov x0, x19 0.00 : ffff800010160e90: bl ffff80001015ff80 0.00 : ffff800010160e94: ldr x20, [x29, #24] 0.00 : ffff800010160e98: b ffff800010160d60 <__rcu_read_unlock+0x40> : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff800010160e9c: dsb sy 0.00 : ffff800010160ea0: ldr x20, [x29, #24] 0.00 : ffff800010160ea4: b ffff800010160d60 <__rcu_read_unlock+0x40> Percent | Source code & Disassembly of vmlinux for cycles (13020 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001014bb20 <__handle_domain_irq>: : __handle_domain_irq(): : * : * Returns: 0 on success, or -EINVAL if conversion has failed : */ : int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq, : bool lookup, struct pt_regs *regs) : { 27.45 : ffff80001014bb20: stp x29, x30, [sp, #-64]! 0.00 : ffff80001014bb24: mov x29, sp 4.79 : ffff80001014bb28: stp x19, x20, [sp, #16] : set_irq_regs(): : : static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs) : { : struct pt_regs *old_regs; : : old_regs = __this_cpu_read(__irq_regs); 0.00 : ffff80001014bb2c: adrp x19, ffff800011528000 : __handle_domain_irq(): 3.06 : ffff80001014bb30: stp x21, x22, [sp, #32] : set_irq_regs(): 0.00 : ffff80001014bb34: add x4, x19, #0x780 : __handle_domain_irq(): 1.32 : ffff80001014bb38: str x23, [sp, #48] 0.00 : ffff80001014bb3c: and w22, w2, #0xff : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.02 : ffff80001014bb40: mrs x5, tpidr_el1 : set_irq_regs(): 0.00 : ffff80001014bb44: ldr x21, [x4, x5] : __handle_domain_irq(): 0.00 : ffff80001014bb48: mov w20, w1 : set_irq_regs(): : __this_cpu_write(__irq_regs, new_regs); 53.40 : ffff80001014bb4c: str x3, [x4, x5] : __handle_domain_irq(): 0.00 : ffff80001014bb50: mov x23, x0 : struct pt_regs *old_regs = set_irq_regs(regs); : unsigned int irq = hwirq; : int ret = 0; : : irq_enter(); 0.00 : ffff80001014bb54: bl ffff8000100ecc58 : : #ifdef CONFIG_IRQ_DOMAIN : if (lookup) 1.21 : ffff80001014bb58: cbnz w22, ffff80001014bba4 <__handle_domain_irq+0x84> : : /* : * Some hardware gives randomly wrong interrupts. Rather : * than crashing, do something sensible. : */ : if (unlikely(!irq || irq >= nr_irqs)) { 0.00 : ffff80001014bb5c: cbz w20, ffff80001014bbb8 <__handle_domain_irq+0x98> 2.83 : ffff80001014bb60: adrp x0, ffff800011921000 0.09 : ffff80001014bb64: ldr w0, [x0, #3096] 0.00 : ffff80001014bb68: cmp w0, w20 0.00 : ffff80001014bb6c: b.ls ffff80001014bbb8 <__handle_domain_irq+0x98> // b.plast : int ret = 0; 0.87 : ffff80001014bb70: mov w22, #0x0 // #0 : ack_bad_irq(irq); : ret = -EINVAL; : } else { : generic_handle_irq(irq); 0.00 : ffff80001014bb74: mov w0, w20 0.00 : ffff80001014bb78: bl ffff80001014b3c8 : } : : irq_exit(); 0.00 : ffff80001014bb7c: bl ffff8000100eccc0 : set_irq_regs(): 0.05 : ffff80001014bb80: add x1, x19, #0x780 : __handle_domain_irq(): : set_irq_regs(old_regs); : return ret; : } 0.00 : ffff80001014bb84: mov w0, w22 : __my_cpu_offset(): 0.02 : ffff80001014bb88: mrs x2, tpidr_el1 : set_irq_regs(): 2.26 : ffff80001014bb8c: str x21, [x1, x2] : __handle_domain_irq(): 0.15 : ffff80001014bb90: ldp x19, x20, [sp, #16] 0.04 : ffff80001014bb94: ldp x21, x22, [sp, #32] 0.00 : ffff80001014bb98: ldr x23, [sp, #48] 2.11 : ffff80001014bb9c: ldp x29, x30, [sp], #64 0.00 : ffff80001014bba0: ret : irq = irq_find_mapping(domain, hwirq); 0.30 : ffff80001014bba4: mov w1, w20 0.00 : ffff80001014bba8: mov x0, x23 0.00 : ffff80001014bbac: bl ffff800010154c60 0.02 : ffff80001014bbb0: mov w20, w0 : if (unlikely(!irq || irq >= nr_irqs)) { 0.00 : ffff80001014bbb4: cbnz w20, ffff80001014bb60 <__handle_domain_irq+0x40> : ack_bad_irq(): : } while (0) : : static inline void ack_bad_irq(unsigned int irq) : { : extern unsigned long irq_err_count; : irq_err_count++; 0.00 : ffff80001014bbb8: adrp x1, ffff800011af2000 : __handle_domain_irq(): : ret = -EINVAL; 0.00 : ffff80001014bbbc: mov w22, #0xffffffea // #-22 : ack_bad_irq(): 0.00 : ffff80001014bbc0: ldr x0, [x1, #152] 0.00 : ffff80001014bbc4: add x0, x0, #0x1 0.00 : ffff80001014bbc8: str x0, [x1, #152] : __handle_domain_irq(): 0.00 : ffff80001014bbcc: b ffff80001014bb7c <__handle_domain_irq+0x5c> Percent | Source code & Disassembly of vmlinux for cycles (12322 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010703840 : : add_interrupt_randomness(): : WRITE_ONCE(f->reg_idx, idx); : return *ptr; : } : : void add_interrupt_randomness(int irq, int irq_flags) : { 0.39 : ffff800010703840: stp x29, x30, [sp, #-96]! : struct entropy_store *r; : struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); : struct pt_regs *regs = get_irq_regs(); : unsigned long now = jiffies; : cycles_t cycles = random_get_entropy(); 0.00 : ffff800010703844: adrp x2, ffff800011ac4000 : { 0.00 : ffff800010703848: mov x29, sp 0.25 : ffff80001070384c: stp x19, x20, [sp, #16] 0.02 : ffff800010703850: adrp x19, ffff800011909000 1.04 : ffff800010703854: stp x21, x22, [sp, #32] 0.01 : ffff800010703858: add x1, x19, #0x908 1.20 : ffff80001070385c: stp x23, x24, [sp, #48] : struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); 0.07 : ffff800010703860: adrp x22, ffff800011527000 : { 0.24 : ffff800010703864: stp x25, x26, [sp, #64] : struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); 0.00 : ffff800010703868: add x22, x22, #0x688 : cycles_t cycles = random_get_entropy(); 0.37 : ffff80001070386c: ldr x2, [x2, #3984] : struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); 0.08 : ffff800010703870: add x22, x22, #0x90 : { 2.96 : ffff800010703874: ldr x3, [x1] 0.90 : ffff800010703878: str x3, [x29, #88] 0.00 : ffff80001070387c: mov x3, #0x0 // #0 0.01 : ffff800010703880: mov x24, x30 : unsigned long now = jiffies; 0.00 : ffff800010703884: adrp x3, ffff800011907000 : get_irq_regs(): : */ : DECLARE_PER_CPU(struct pt_regs *, __irq_regs); : : static inline struct pt_regs *get_irq_regs(void) : { : return __this_cpu_read(__irq_regs); 0.22 : ffff800010703888: adrp x1, ffff800011528000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.37 : ffff80001070388c: mrs x23, tpidr_el1 : get_irq_regs(): 1.06 : ffff800010703890: add x1, x1, #0x780 : add_interrupt_randomness(): : { 0.11 : ffff800010703894: mov w25, w0 : get_irq_regs(): 0.88 : ffff800010703898: ldr x26, [x1, x23] : add_interrupt_randomness(): : struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); 0.00 : ffff80001070389c: add x20, x22, x23 : unsigned long now = jiffies; 1.57 : ffff8000107038a0: ldr x21, [x3, #2432] : cycles_t cycles = random_get_entropy(); 0.30 : ffff8000107038a4: blr x2 : __u32 c_high, j_high; : __u64 ip; : unsigned long seed; : int credit = 0; : : if (cycles == 0) 50.72 : ffff8000107038a8: cbz x0, ffff8000107039fc 0.00 : ffff8000107038ac: mov w1, w0 0.00 : ffff8000107038b0: lsr x2, x0, #32 : cycles = get_reg(fast_pool, regs); : c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0; : j_high = (sizeof(now) > 4) ? now >> 32 : 0; : fast_pool->pool[0] ^= cycles ^ j_high ^ irq; 0.00 : ffff8000107038b4: ldr w0, [x22, x23] : j_high = (sizeof(now) > 4) ? now >> 32 : 0; 0.00 : ffff8000107038b8: lsr x3, x21, #32 : fast_pool->pool[0] ^= cycles ^ j_high ^ irq; 0.00 : ffff8000107038bc: eor w25, w25, w3 : fast_pool->pool[1] ^= now ^ c_high; : ip = regs ? instruction_pointer(regs) : _RET_IP_; 0.00 : ffff8000107038c0: mov x30, x24 : fast_pool->pool[0] ^= cycles ^ j_high ^ irq; 0.01 : ffff8000107038c4: eor w0, w1, w0 0.00 : ffff8000107038c8: eor w0, w0, w25 2.45 : ffff8000107038cc: str w0, [x22, x23] : fast_pool->pool[1] ^= now ^ c_high; 0.00 : ffff8000107038d0: ldr w1, [x20, #4] 0.00 : ffff8000107038d4: eor w1, w1, w21 0.00 : ffff8000107038d8: eor w1, w1, w2 0.00 : ffff8000107038dc: str w1, [x20, #4] : ip = regs ? instruction_pointer(regs) : _RET_IP_; 0.00 : ffff8000107038e0: cbz x26, ffff8000107038e8 0.00 : ffff8000107038e4: ldr x30, [x26, #256] : fast_pool->pool[2] ^= ip; : fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 : 2.82 : ffff8000107038e8: ldp w3, w2, [x20, #8] 0.00 : ffff8000107038ec: lsr x4, x30, #32 : fast_mix(): : a += b; c += d; 0.00 : ffff8000107038f0: add w0, w0, w1 : add_interrupt_randomness(): : get_reg(fast_pool, regs); : : fast_mix(fast_pool); : add_interrupt_bench(cycles); : : if (unlikely(crng_init == 0)) { 0.00 : ffff8000107038f4: adrp x5, ffff800011b4c000 : fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 : 0.00 : ffff8000107038f8: eor w2, w2, w4 : fast_pool->pool[2] ^= ip; 0.00 : ffff8000107038fc: eor w3, w3, w30 : fast_mix(): : a += b; c += d; 0.00 : ffff800010703900: add w3, w3, w2 : add_interrupt_randomness(): : if (unlikely(crng_init == 0)) { 0.00 : ffff800010703904: ldr w5, [x5, #1312] : fast_mix(): : d ^= a; b ^= c; 0.00 : ffff800010703908: eor w2, w0, w2, ror #5 0.00 : ffff80001070390c: eor w1, w3, w1, ror #26 : a += b; c += d; 0.00 : ffff800010703910: add w0, w0, w1 4.06 : ffff800010703914: add w3, w3, w2 : d ^= a; b ^= c; 0.00 : ffff800010703918: eor w1, w3, w1, ror #16 0.00 : ffff80001070391c: eor w2, w0, w2, ror #18 : a += b; c += d; 0.00 : ffff800010703920: add w3, w3, w2 1.91 : ffff800010703924: add w0, w0, w1 : d ^= a; b ^= c; 0.00 : ffff800010703928: eor w1, w3, w1, ror #26 0.00 : ffff80001070392c: eor w2, w0, w2, ror #5 : a += b; c += d; 0.00 : ffff800010703930: add w0, w0, w1 : f->pool[0] = a; f->pool[1] = b; 0.00 : ffff800010703934: str w0, [x22, x23] : a += b; c += d; 0.00 : ffff800010703938: add w3, w3, w2 : d ^= a; b ^= c; 0.00 : ffff80001070393c: eor w0, w0, w2, ror #18 : f->count++; 0.00 : ffff800010703940: ldrb w4, [x20, #26] : d ^= a; b ^= c; 0.00 : ffff800010703944: eor w1, w3, w1, ror #16 : f->pool[2] = c; f->pool[3] = d; 8.87 : ffff800010703948: stp w1, w3, [x20, #4] : f->count++; 0.00 : ffff80001070394c: add w2, w4, #0x1 : f->pool[2] = c; f->pool[3] = d; 1.35 : ffff800010703950: str w0, [x20, #12] : f->count++; 0.00 : ffff800010703954: and w1, w2, #0xff 0.00 : ffff800010703958: strb w1, [x20, #26] : add_interrupt_randomness(): : if ((fast_pool->count >= 64) && 0.00 : ffff80001070395c: cmp w1, #0x3f : if (unlikely(crng_init == 0)) { 0.00 : ffff800010703960: cbz w5, ffff800010703a3c : fast_pool->last = now; : } : return; : } : : if ((fast_pool->count < 64) && 1.37 : ffff800010703964: b.hi ffff8000107039a8 // b.pmore : !time_after(now, fast_pool->last + HZ)) 0.42 : ffff800010703968: ldr x1, [x20, #16] 0.00 : ffff80001070396c: mov x0, #0xfa // #250 0.00 : ffff800010703970: sub x0, x0, x21 0.00 : ffff800010703974: cmn x0, x1 5.60 : ffff800010703978: b.mi ffff8000107039a8 // b.first : : fast_pool->count = 0; : : /* award one bit for the contents of the fast pool */ : credit_entropy_bits(r, credit + 1); : } 0.00 : ffff80001070397c: add x19, x19, #0x908 0.25 : ffff800010703980: ldr x1, [x29, #88] 0.43 : ffff800010703984: ldr x0, [x19] 0.00 : ffff800010703988: eor x0, x1, x0 0.00 : ffff80001070398c: cbnz x0, ffff800010703a98 2.83 : ffff800010703990: ldp x19, x20, [sp, #16] 0.99 : ffff800010703994: ldp x21, x22, [sp, #32] 0.03 : ffff800010703998: ldp x23, x24, [sp, #48] 0.79 : ffff80001070399c: ldp x25, x26, [sp, #64] 1.99 : ffff8000107039a0: ldp x29, x30, [sp], #96 0.67 : ffff8000107039a4: ret : spin_trylock(): : raw_spin_lock_bh(&lock->rlock); : } : : static __always_inline int spin_trylock(spinlock_t *lock) : { : return raw_spin_trylock(&lock->rlock); 0.16 : ffff8000107039a8: adrp x22, ffff800011a87000 0.00 : ffff8000107039ac: add x0, x22, #0xc18 0.00 : ffff8000107039b0: add x23, x0, #0x30 0.00 : ffff8000107039b4: add x0, x0, #0x48 0.10 : ffff8000107039b8: bl ffff800010cd7fe0 <_raw_spin_trylock> : add_interrupt_randomness(): : if (!spin_trylock(&r->lock)) 0.10 : ffff8000107039bc: cbz w0, ffff80001070397c : fast_pool->last = now; 0.00 : ffff8000107039c0: str x21, [x20, #16] : __mix_pool_bytes(): : _mix_pool_bytes(r, in, nbytes); 0.00 : ffff8000107039c4: mov w2, #0x10 // #16 0.00 : ffff8000107039c8: mov x1, x20 0.00 : ffff8000107039cc: mov x0, x23 0.00 : ffff8000107039d0: bl ffff8000107017a0 <_mix_pool_bytes> : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000107039d4: b ffff800010703a5c : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff8000107039d8: nop 0.02 : ffff8000107039dc: mov w21, #0x1 // #1 : spin_unlock(): : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ : } while (0) : : static __always_inline void spin_unlock(spinlock_t *lock) : { : raw_spin_unlock(&lock->rlock); 0.00 : ffff8000107039e0: add x0, x22, #0xc18 0.00 : ffff8000107039e4: add x0, x0, #0x48 0.00 : ffff8000107039e8: bl ffff800010cd7c38 <_raw_spin_unlock> : add_interrupt_randomness(): : fast_pool->count = 0; 0.00 : ffff8000107039ec: strb wzr, [x20, #26] : credit_entropy_bits(r, credit + 1); 0.00 : ffff8000107039f0: mov w0, w21 0.00 : ffff8000107039f4: bl ffff800010702a00 0.02 : ffff8000107039f8: b ffff80001070397c : get_reg(): : if (regs == NULL) 0.00 : ffff8000107039fc: mov w1, #0x0 // #0 0.00 : ffff800010703a00: cbz x26, ffff800010703a28 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010703a04: ldrh w0, [x20, #24] 0.00 : ffff800010703a08: and w0, w0, #0xffff : get_reg(): : if (idx >= sizeof(struct pt_regs) / sizeof(__u32)) 0.00 : ffff800010703a0c: cmp w0, #0x4f 0.00 : ffff800010703a10: b.hi ffff800010703a30 // b.pmore 0.00 : ffff800010703a14: add w1, w0, #0x1 0.00 : ffff800010703a18: add x0, x26, w0, uxth #2 0.00 : ffff800010703a1c: and w1, w1, #0xffff : __write_once_size(): : : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; 0.00 : ffff800010703a20: strh w1, [x20, #24] : get_reg(): : return *ptr; 0.00 : ffff800010703a24: ldr w1, [x0] : if (regs == NULL) 0.00 : ffff800010703a28: mov w2, #0x0 // #0 0.00 : ffff800010703a2c: b ffff8000107038b4 : if (idx >= sizeof(struct pt_regs) / sizeof(__u32)) 0.00 : ffff800010703a30: mov x0, x26 0.00 : ffff800010703a34: mov w1, #0x1 // #1 0.00 : ffff800010703a38: b ffff800010703a20 : add_interrupt_randomness(): : if ((fast_pool->count >= 64) && 0.00 : ffff800010703a3c: b.ls ffff80001070397c // b.plast : crng_fast_load((char *) fast_pool->pool, 0.00 : ffff800010703a40: mov x1, #0x10 // #16 0.00 : ffff800010703a44: mov x0, x20 0.00 : ffff800010703a48: bl ffff800010703598 : if ((fast_pool->count >= 64) && 0.00 : ffff800010703a4c: cbz w0, ffff80001070397c : fast_pool->last = now; 0.00 : ffff800010703a50: str x21, [x20, #16] : fast_pool->count = 0; 0.00 : ffff800010703a54: strb wzr, [x20, #26] 0.00 : ffff800010703a58: b ffff80001070397c : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010703a5c: adrp x0, ffff800011af2000 0.00 : ffff800010703a60: ldr x0, [x0, #1320] : arch_get_random_seed_long(): : * Only support the generic interface after we have detected : * the system wide capability, avoiding complexity with the : * cpufeature code and with potential scheduling between CPUs : * with and without the feature. : */ : if (!cpus_have_const_cap(ARM64_HAS_RNG)) 0.00 : ffff800010703a64: tbz x0, #50, ffff8000107039dc : __arm64_rndr(): 0.00 : ffff800010703a68: mrs x1, s3_3_c2_c4_0 : asm volatile( 0.00 : ffff800010703a6c: cset w0, ne // ne = any 0.00 : ffff800010703a70: str x1, [x29, #80] : add_interrupt_randomness(): : if (arch_get_random_seed_long(&seed)) { 0.00 : ffff800010703a74: tst w0, #0xff 0.00 : ffff800010703a78: b.eq ffff8000107039dc // b.none : __mix_pool_bytes(): : _mix_pool_bytes(r, in, nbytes); 0.00 : ffff800010703a7c: add x0, x22, #0xc18 0.00 : ffff800010703a80: mov w21, #0x2 // #2 0.00 : ffff800010703a84: mov w2, #0x8 // #8 0.00 : ffff800010703a88: add x1, x29, #0x50 0.00 : ffff800010703a8c: add x0, x0, #0x30 0.00 : ffff800010703a90: bl ffff8000107017a0 <_mix_pool_bytes> 0.00 : ffff800010703a94: b ffff8000107039e0 : add_interrupt_randomness(): : } 0.00 : ffff800010703a98: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (11242 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001071d5e0 : : arm_smmu_cmdq_build_cmd(): : } : : /* High-level queue accessors */ : static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) : { : memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT); 1.84 : ffff80001071d5e0: stp xzr, xzr, [x0] : cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode); 21.63 : ffff80001071d5e4: ldr x3, [x0] 23.69 : ffff80001071d5e8: ldrb w2, [x1] 0.00 : ffff80001071d5ec: orr x2, x2, x3 1.67 : ffff80001071d5f0: str x2, [x0] : : switch (ent->opcode) { 3.04 : ffff80001071d5f4: ldrb w3, [x1] 0.05 : ffff80001071d5f8: cmp w3, #0x12 0.00 : ffff80001071d5fc: b.eq ffff80001071d858 // b.none 0.06 : ffff80001071d600: b.ls ffff80001071d630 // b.plast 0.73 : ffff80001071d604: cmp w3, #0x30 0.21 : ffff80001071d608: b.eq ffff80001071d7cc // b.none 1.42 : ffff80001071d60c: b.ls ffff80001071d748 // b.plast 1.09 : ffff80001071d610: cmp w3, #0x41 0.00 : ffff80001071d614: b.eq ffff80001071d674 // b.none 0.20 : ffff80001071d618: cmp w3, #0x46 0.00 : ffff80001071d61c: b.eq ffff80001071d7f8 // b.none 0.00 : ffff80001071d620: cmp w3, #0x40 0.00 : ffff80001071d624: b.eq ffff80001071d6e4 // b.none : } : cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH); : cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); : break; : default: : return -ENOENT; 0.00 : ffff80001071d628: mov w3, #0xfffffffe // #-2 0.00 : ffff80001071d62c: b ffff80001071d66c : switch (ent->opcode) { 0.00 : ffff80001071d630: cmp w3, #0x4 0.00 : ffff80001071d634: b.eq ffff80001071d844 // b.none 0.00 : ffff80001071d638: b.ls ffff80001071d78c // b.plast 0.00 : ffff80001071d63c: cmp w3, #0x6 0.00 : ffff80001071d640: b.eq ffff80001071d6d0 // b.none 0.00 : ffff80001071d644: b.cc ffff80001071d830 // b.lo, b.ul, b.last 0.00 : ffff80001071d648: cmp w3, #0x11 0.00 : ffff80001071d64c: b.ne ffff80001071d628 // b.any : cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); 0.00 : ffff80001071d650: ldrh w3, [x1, #8] 0.00 : ffff80001071d654: orr x2, x2, x3, lsl #48 0.00 : ffff80001071d658: str x2, [x0] : cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 0.00 : ffff80001071d65c: ldrh w1, [x1, #10] : } : : return 0; 0.00 : ffff80001071d660: mov w3, #0x0 // #0 : cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 0.00 : ffff80001071d664: orr x2, x2, x1, lsl #32 0.00 : ffff80001071d668: str x2, [x0] : } 2.11 : ffff80001071d66c: mov w0, w3 0.00 : ffff80001071d670: ret : cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 0.00 : ffff80001071d674: ldrb w3, [x1, #1] : cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid); 0.00 : ffff80001071d678: ldr x4, [x0, #8] : cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 0.00 : ffff80001071d67c: orr x3, x2, x3, lsl #11 0.00 : ffff80001071d680: str x3, [x0] : cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid); 0.00 : ffff80001071d684: ldr w2, [x1, #12] 0.00 : ffff80001071d688: lsl w2, w2, #12 0.00 : ffff80001071d68c: orr x2, x2, x3 0.00 : ffff80001071d690: str x2, [x0] : cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid); 0.00 : ffff80001071d694: ldr w3, [x1, #8] 0.00 : ffff80001071d698: orr x2, x2, x3, lsl #32 0.00 : ffff80001071d69c: str x2, [x0] : cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid); 0.00 : ffff80001071d6a0: ldrh w2, [x1, #16] 0.00 : ffff80001071d6a4: and x2, x2, #0x1ff 0.00 : ffff80001071d6a8: orr x2, x2, x4 0.00 : ffff80001071d6ac: str x2, [x0, #8] : switch (ent->pri.resp) { 0.00 : ffff80001071d6b0: ldr w1, [x1, #20] 0.00 : ffff80001071d6b4: cmp w1, #0x2 0.00 : ffff80001071d6b8: b.hi ffff80001071d898 // b.pmore : cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp); 0.00 : ffff80001071d6bc: ubfiz x1, x1, #12, #32 : return 0; 0.00 : ffff80001071d6c0: mov w3, #0x0 // #0 : cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp); 0.00 : ffff80001071d6c4: orr x2, x1, x2 0.00 : ffff80001071d6c8: str x2, [x0, #8] : break; 0.00 : ffff80001071d6cc: b ffff80001071d66c : cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); 0.00 : ffff80001071d6d0: ldr w1, [x1, #8] : return 0; 0.00 : ffff80001071d6d4: mov w3, #0x0 // #0 : cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); 0.00 : ffff80001071d6d8: orr x2, x2, x1, lsl #32 0.00 : ffff80001071d6dc: str x2, [x0] : break; 0.00 : ffff80001071d6e0: b ffff80001071d66c : cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 0.00 : ffff80001071d6e4: ldrb w4, [x1, #1] : return 0; 0.00 : ffff80001071d6e8: mov w3, #0x0 // #0 : cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size); 0.00 : ffff80001071d6ec: ldr x6, [x0, #8] : cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); 0.00 : ffff80001071d6f0: orr x2, x2, x4, lsl #11 0.00 : ffff80001071d6f4: str x2, [x0] : cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global); 0.00 : ffff80001071d6f8: ldrb w4, [x1, #25] 0.00 : ffff80001071d6fc: orr x2, x2, x4, lsl #9 0.00 : ffff80001071d700: str x2, [x0] : cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid); 0.00 : ffff80001071d704: ldr w4, [x1, #12] 0.00 : ffff80001071d708: lsl w4, w4, #12 0.00 : ffff80001071d70c: orr x4, x4, x2 0.00 : ffff80001071d710: str x4, [x0] : cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid); 0.00 : ffff80001071d714: ldr w5, [x1, #8] 0.00 : ffff80001071d718: orr x4, x4, x5, lsl #32 0.00 : ffff80001071d71c: str x4, [x0] : cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size); 0.00 : ffff80001071d720: ldrb w2, [x1, #24] 0.00 : ffff80001071d724: and x2, x2, #0x3f 0.00 : ffff80001071d728: orr x2, x2, x6 0.00 : ffff80001071d72c: str x2, [x0, #8] : cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK; 0.00 : ffff80001071d730: ldr x1, [x1, #16] 0.00 : ffff80001071d734: and x1, x1, #0xfffffffffffff000 0.00 : ffff80001071d738: orr x2, x1, x2 0.00 : ffff80001071d73c: str x2, [x0, #8] : } 0.00 : ffff80001071d740: mov w0, w3 0.00 : ffff80001071d744: ret : switch (ent->opcode) { 0.00 : ffff80001071d748: cmp w3, #0x28 0.00 : ffff80001071d74c: b.eq ffff80001071d65c // b.none 0.00 : ffff80001071d750: cmp w3, #0x2a 0.00 : ffff80001071d754: b.ne ffff80001071d7c4 // b.any : cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 0.00 : ffff80001071d758: ldrh w4, [x1, #10] : return 0; 0.00 : ffff80001071d75c: mov w3, #0x0 // #0 : cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); 0.00 : ffff80001071d760: ldr x5, [x0, #8] : cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 0.00 : ffff80001071d764: orr x2, x2, x4, lsl #32 0.00 : ffff80001071d768: str x2, [x0] : cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); 0.00 : ffff80001071d76c: ldrb w2, [x1, #12] 0.00 : ffff80001071d770: orr x2, x2, x5 0.00 : ffff80001071d774: str x2, [x0, #8] : cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK; 0.00 : ffff80001071d778: ldr x1, [x1, #16] 0.00 : ffff80001071d77c: and x1, x1, #0xffffffffff000 0.00 : ffff80001071d780: orr x1, x1, x2 0.00 : ffff80001071d784: str x1, [x0, #8] : break; 0.00 : ffff80001071d788: b ffff80001071d66c : switch (ent->opcode) { 0.00 : ffff80001071d78c: cmp w3, #0x1 0.00 : ffff80001071d790: b.eq ffff80001071d7d4 // b.none 0.00 : ffff80001071d794: cmp w3, #0x3 0.00 : ffff80001071d798: b.ne ffff80001071d628 // b.any : cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); 0.00 : ffff80001071d79c: ldr w5, [x1, #8] : return 0; 0.00 : ffff80001071d7a0: mov w3, #0x0 // #0 : cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf); 0.00 : ffff80001071d7a4: ldr x4, [x0, #8] : cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); 0.00 : ffff80001071d7a8: orr x2, x2, x5, lsl #32 0.00 : ffff80001071d7ac: str x2, [x0] : cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf); 0.00 : ffff80001071d7b0: ldrb w1, [x1, #16] 0.00 : ffff80001071d7b4: orr x1, x4, x1 0.00 : ffff80001071d7b8: str x1, [x0, #8] : } 0.00 : ffff80001071d7bc: mov w0, w3 0.00 : ffff80001071d7c0: ret : switch (ent->opcode) { 0.00 : ffff80001071d7c4: cmp w3, #0x20 0.00 : ffff80001071d7c8: b.ne ffff80001071d628 // b.any : return 0; 0.00 : ffff80001071d7cc: mov w3, #0x0 // #0 0.00 : ffff80001071d7d0: b ffff80001071d66c : cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid); 0.00 : ffff80001071d7d4: ldr w4, [x1, #8] : return 0; 0.00 : ffff80001071d7d8: mov w3, #0x0 // #0 : cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size); 0.00 : ffff80001071d7dc: ldr x5, [x0, #8] : cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid); 0.00 : ffff80001071d7e0: orr x2, x2, x4, lsl #32 0.00 : ffff80001071d7e4: str x2, [x0] : cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size); 0.00 : ffff80001071d7e8: ldrb w2, [x1, #12] 0.00 : ffff80001071d7ec: and x2, x2, #0x1f 0.00 : ffff80001071d7f0: orr x2, x2, x5 0.00 : ffff80001071d7f4: b ffff80001071d72c : if (ent->sync.msiaddr) { 0.52 : ffff80001071d7f8: ldr x4, [x1, #8] : cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV); 0.00 : ffff80001071d7fc: orr x3, x2, #0x2000 : if (ent->sync.msiaddr) { 0.61 : ffff80001071d800: cbz x4, ffff80001071d820 : cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ); 0.95 : ffff80001071d804: orr x3, x2, #0x1000 0.95 : ffff80001071d808: str x3, [x0] : cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK; 0.62 : ffff80001071d80c: ldr x2, [x0, #8] 0.59 : ffff80001071d810: ldr x1, [x1, #8] 0.00 : ffff80001071d814: and x1, x1, #0xffffffffffffc 0.03 : ffff80001071d818: orr x1, x2, x1 0.91 : ffff80001071d81c: str x1, [x0, #8] : cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); 0.15 : ffff80001071d820: orr x1, x3, #0xfc00000 : return 0; 0.00 : ffff80001071d824: mov w3, #0x0 // #0 : cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); 1.46 : ffff80001071d828: str x1, [x0] : break; 0.00 : ffff80001071d82c: b ffff80001071d66c : cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid); 0.00 : ffff80001071d830: ldr w3, [x1, #12] 0.00 : ffff80001071d834: lsl w3, w3, #12 0.00 : ffff80001071d838: orr x2, x2, x3 0.00 : ffff80001071d83c: str x2, [x0] 0.00 : ffff80001071d840: b ffff80001071d79c : cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); 0.00 : ffff80001071d844: ldr x1, [x0, #8] : return 0; 0.00 : ffff80001071d848: mov w3, #0x0 // #0 : cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); 0.00 : ffff80001071d84c: orr x1, x1, #0x1f 0.00 : ffff80001071d850: str x1, [x0, #8] : break; 0.00 : ffff80001071d854: b ffff80001071d66c : cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 0.18 : ffff80001071d858: ldrh w4, [x1, #10] : return 0; 0.00 : ffff80001071d85c: mov w3, #0x0 // #0 : cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); 11.54 : ffff80001071d860: ldr x5, [x0, #8] : cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); 0.00 : ffff80001071d864: orr x2, x2, x4, lsl #32 8.00 : ffff80001071d868: str x2, [x0] : cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); 0.89 : ffff80001071d86c: ldrh w4, [x1, #8] 0.00 : ffff80001071d870: orr x2, x2, x4, lsl #48 1.94 : ffff80001071d874: str x2, [x0] : cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); 0.96 : ffff80001071d878: ldrb w2, [x1, #12] 0.00 : ffff80001071d87c: orr x2, x2, x5 2.04 : ffff80001071d880: str x2, [x0, #8] : cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK; 5.25 : ffff80001071d884: ldr x1, [x1, #16] 0.00 : ffff80001071d888: and x1, x1, #0xfffffffffffff000 0.00 : ffff80001071d88c: orr x1, x1, x2 4.67 : ffff80001071d890: str x1, [x0, #8] : break; 0.00 : ffff80001071d894: b ffff80001071d66c : return -EINVAL; 0.00 : ffff80001071d898: mov w3, #0xffffffea // #-22 0.00 : ffff80001071d89c: b ffff80001071d66c Percent | Source code & Disassembly of vmlinux for cycles (10971 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010713aa0 <__iommu_dma_unmap>: : __iommu_dma_unmap(): : size >> iova_shift(iovad)); : } : : static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr, : size_t size) : { 1.04 : ffff800010713aa0: stp x29, x30, [sp, #-96]! 0.13 : ffff800010713aa4: mov x29, sp 0.35 : ffff800010713aa8: stp x19, x20, [sp, #16] 0.06 : ffff800010713aac: mov x20, x1 0.60 : ffff800010713ab0: stp x21, x22, [sp, #32] 0.14 : ffff800010713ab4: adrp x21, ffff800011909000 1.07 : ffff800010713ab8: stp x23, x24, [sp, #48] 0.00 : ffff800010713abc: add x3, x21, #0x908 0.03 : ffff800010713ac0: mov x19, x2 : iommu_iotlb_gather_init(): : return (struct iommu_device *)dev_get_drvdata(dev); : } : : static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) : { : *gather = (struct iommu_iotlb_gather) { 0.00 : ffff800010713ac4: add x22, x29, #0x60 : __iommu_dma_unmap(): 0.75 : ffff800010713ac8: ldr x1, [x3] 0.51 : ffff800010713acc: str x1, [x29, #88] 0.16 : ffff800010713ad0: mov x1, #0x0 // #0 : iova_align(): : return iova & iova_mask(iovad); : } : : static inline size_t iova_align(struct iova_domain *iovad, size_t size) : { : return ALIGN(size, iovad->granule); 0.01 : ffff800010713ad4: sub x19, x19, #0x1 : __iommu_dma_unmap(): : struct iommu_domain *domain = iommu_get_dma_domain(dev); 0.01 : ffff800010713ad8: bl ffff800010712818 : struct iommu_dma_cookie *cookie = domain->iova_cookie; 10.23 : ffff800010713adc: ldr x23, [x0, #64] : iommu_iotlb_gather_init(): 1.50 : ffff800010713ae0: mov x1, #0xffffffffffffffff // #-1 5.63 : ffff800010713ae4: str x1, [x22, #-32]! : __iommu_dma_unmap(): : struct iommu_domain *domain = iommu_get_dma_domain(dev); 0.01 : ffff800010713ae8: mov x24, x0 : iommu_iotlb_gather_init(): 0.32 : ffff800010713aec: stp xzr, xzr, [x29, #72] : iova_offset(): : return iova & iova_mask(iovad); 0.45 : ffff800010713af0: ldr x2, [x23, #40] : __iommu_dma_unmap(): : : dma_addr -= iova_off; : size = iova_align(iovad, size + iova_off); : iommu_iotlb_gather_init(&iotlb_gather); : : unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather); 2.75 : ffff800010713af4: mov x3, x22 : iova_mask(): : return iovad->granule - 1; 0.23 : ffff800010713af8: add x1, x2, x1 : iova_align(): : return ALIGN(size, iovad->granule); 0.36 : ffff800010713afc: neg x4, x2 : iova_offset(): : return iova & iova_mask(iovad); 30.12 : ffff800010713b00: and x1, x1, x20 : iova_align(): : return ALIGN(size, iovad->granule); 0.02 : ffff800010713b04: add x2, x2, x1 : __iommu_dma_unmap(): : dma_addr -= iova_off; 0.04 : ffff800010713b08: sub x20, x20, x1 : iova_align(): 0.00 : ffff800010713b0c: add x19, x19, x2 : __iommu_dma_unmap(): : unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather); 0.18 : ffff800010713b10: mov x1, x20 : iova_align(): 0.05 : ffff800010713b14: and x19, x19, x4 : __iommu_dma_unmap(): 0.00 : ffff800010713b18: mov x2, x19 0.01 : ffff800010713b1c: bl ffff800010711948 : WARN_ON(unmapped != size); 1.72 : ffff800010713b20: cmp x0, x19 0.00 : ffff800010713b24: b.ne ffff800010713b90 <__iommu_dma_unmap+0xf0> // b.any : : if (!cookie->fq_domain) 0.91 : ffff800010713b28: ldr x0, [x23, #1896] 1.27 : ffff800010713b2c: cbz x0, ffff800010713b68 <__iommu_dma_unmap+0xc8> : iommu_tlb_sync(domain, &iotlb_gather); : iommu_dma_free_iova(cookie, dma_addr, size); 3.80 : ffff800010713b30: mov x1, x20 0.00 : ffff800010713b34: mov x0, x23 0.00 : ffff800010713b38: mov x2, x19 : } 0.00 : ffff800010713b3c: add x21, x21, #0x908 : iommu_dma_free_iova(cookie, dma_addr, size); 0.07 : ffff800010713b40: bl ffff800010713a40 : } 0.23 : ffff800010713b44: ldr x1, [x29, #88] 0.47 : ffff800010713b48: ldr x0, [x21] 0.00 : ffff800010713b4c: eor x0, x1, x0 0.00 : ffff800010713b50: cbnz x0, ffff800010713b98 <__iommu_dma_unmap+0xf8> 3.57 : ffff800010713b54: ldp x19, x20, [sp, #16] 1.16 : ffff800010713b58: ldp x21, x22, [sp, #32] 0.01 : ffff800010713b5c: ldp x23, x24, [sp, #48] 5.30 : ffff800010713b60: ldp x29, x30, [sp], #96 0.00 : ffff800010713b64: ret : iommu_tlb_sync(): : } : : static inline void iommu_tlb_sync(struct iommu_domain *domain, : struct iommu_iotlb_gather *iotlb_gather) : { : if (domain->ops->iotlb_sync) 2.41 : ffff800010713b68: ldr x0, [x24, #8] 5.60 : ffff800010713b6c: ldr x2, [x0, #72] 0.00 : ffff800010713b70: cbz x2, ffff800010713b80 <__iommu_dma_unmap+0xe0> : domain->ops->iotlb_sync(domain, iotlb_gather); 13.14 : ffff800010713b74: mov x1, x22 0.00 : ffff800010713b78: mov x0, x24 0.00 : ffff800010713b7c: blr x2 : iommu_iotlb_gather_init(): : *gather = (struct iommu_iotlb_gather) { 0.84 : ffff800010713b80: mov x0, #0xffffffffffffffff // #-1 1.50 : ffff800010713b84: str x0, [x29, #64] 1.27 : ffff800010713b88: stp xzr, xzr, [x29, #72] 0.00 : ffff800010713b8c: b ffff800010713b30 <__iommu_dma_unmap+0x90> : __iommu_dma_unmap(): : WARN_ON(unmapped != size); 0.00 : ffff800010713b90: brk #0x800 0.00 : ffff800010713b94: b ffff800010713b28 <__iommu_dma_unmap+0x88> : } 0.00 : ffff800010713b98: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (10748 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010115e18 : : check_preempt_curr(): : } else if (oldprio != p->prio || dl_task(p)) : p->sched_class->prio_changed(rq, p, oldprio); : } : : void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) : { 0.81 : ffff800010115e18: stp x29, x30, [sp, #-32]! 0.01 : ffff800010115e1c: mov x29, sp 1.04 : ffff800010115e20: str x19, [sp, #16] 0.00 : ffff800010115e24: mov x19, x0 : const struct sched_class *class; : : if (p->sched_class == rq->curr->sched_class) { 1.41 : ffff800010115e28: ldr x4, [x0, #2352] 1.29 : ffff800010115e2c: ldr x3, [x1, #128] 1.05 : ffff800010115e30: ldr x5, [x4, #128] 3.41 : ffff800010115e34: cmp x3, x5 10.51 : ffff800010115e38: b.ne ffff800010115e7c // b.any : rq->curr->sched_class->check_preempt_curr(rq, p, flags); 0.04 : ffff800010115e3c: ldr x3, [x3, #40] 0.00 : ffff800010115e40: blr x3 0.05 : ffff800010115e44: ldr x4, [x19, #2352] : : /* : * A queue event has occurred, and we're going to schedule. In : * this case, we can save a useless back to back clock update. : */ : if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 2.29 : ffff800010115e48: ldr w0, [x4, #104] 0.00 : ffff800010115e4c: cmp w0, #0x1 0.00 : ffff800010115e50: b.eq ffff800010115e60 // b.none : rq_clock_skip_update(rq); : } 0.04 : ffff800010115e54: ldr x19, [sp, #16] 0.04 : ffff800010115e58: ldp x29, x30, [sp], #32 0.00 : ffff800010115e5c: ret : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 19.38 : ffff800010115e60: ldr x0, [x4] : check_preempt_curr(): : if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 0.00 : ffff800010115e64: tst w0, #0x2 0.00 : ffff800010115e68: b.eq ffff800010115e54 // b.none : rq_clock_skip_update(): : } : : static inline void rq_clock_skip_update(struct rq *rq) : { : lockdep_assert_held(&rq->lock); : rq->clock_update_flags |= RQCF_REQ_SKIP; 8.64 : ffff800010115e6c: ldr w0, [x19, #2392] 0.00 : ffff800010115e70: orr w0, w0, #0x1 2.64 : ffff800010115e74: str w0, [x19, #2392] : check_preempt_curr(): : } 0.00 : ffff800010115e78: b ffff800010115e54 : if (class == rq->curr->sched_class) 30.38 : ffff800010115e7c: adrp x0, ffff800010d06000 0.01 : ffff800010115e80: add x0, x0, #0x388 0.13 : ffff800010115e84: cmp x0, #0x0 0.10 : ffff800010115e88: ccmp x5, x0, #0x4, ne // ne = any 2.44 : ffff800010115e8c: b.ne ffff800010115ea4 // b.any 0.00 : ffff800010115e90: b ffff800010115e48 : for_each_class(class) { 6.31 : ffff800010115e94: ldr x0, [x0] : if (class == rq->curr->sched_class) 0.00 : ffff800010115e98: cmp x0, #0x0 0.00 : ffff800010115e9c: ccmp x5, x0, #0x4, ne // ne = any 0.06 : ffff800010115ea0: b.eq ffff800010115e48 // b.none : if (class == p->sched_class) { 3.02 : ffff800010115ea4: cmp x3, x0 0.00 : ffff800010115ea8: b.ne ffff800010115e94 // b.any : resched_curr(rq); 2.79 : ffff800010115eac: mov x0, x19 0.00 : ffff800010115eb0: bl ffff800010114bb8 2.11 : ffff800010115eb4: ldr x4, [x19, #2352] : break; 0.00 : ffff800010115eb8: b ffff800010115e48 Percent | Source code & Disassembly of vmlinux for cycles (10592 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff800010463dd8 : : blk_mq_tag_to_rq(): : } : EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); : : struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) : { : if (tag < tags->nr_tags) { 0.07 : ffff800010463dd8: ldr w2, [x0] 0.00 : ffff800010463ddc: cmp w2, w1 0.63 : ffff800010463de0: b.ls ffff800010463df4 // b.plast : prefetch(tags->rqs[tag]); 14.34 : ffff800010463de4: ldr x0, [x0, #144] 25.90 : ffff800010463de8: ldr x0, [x0, w1, uxtw #3] : prefetch(): : * Prefetching support : */ : #define ARCH_HAS_PREFETCH : static inline void prefetch(const void *ptr) : { : asm volatile("prfm pldl1keep, %a0\n" : : "p" (ptr)); 59.06 : ffff800010463dec: prfm pldl1keep, [x0] : blk_mq_tag_to_rq(): : return tags->rqs[tag]; : } : : return NULL; : } 0.00 : ffff800010463df0: ret : return NULL; 0.00 : ffff800010463df4: mov x0, #0x0 // #0 : } 0.00 : ffff800010463df8: ret Percent | Source code & Disassembly of vmlinux for cycles (10059 samples, percent: local period) ------------------------------------------------------------------------------------------------------ : : : : Disassembly of section .text: : : ffff80001045bb30 : : blk_queue_exit(): : return -ENODEV; : } : } : : void blk_queue_exit(struct request_queue *q) : { 0.01 : ffff80001045bb30: stp x29, x30, [sp, #-32]! 0.00 : ffff80001045bb34: mov x29, sp 0.13 : ffff80001045bb38: str x19, [sp, #16] 0.00 : ffff80001045bb3c: mov x19, x0 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001045bb40: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.06 : ffff80001045bb44: ldr x1, [x19, #1480] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001045bb48: tst x1, #0x3 0.00 : ffff80001045bb4c: b.ne ffff80001045bbbc // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 1.30 : ffff80001045bb50: mrs x2, sp_el0 : __read_once_size(): 1.91 : ffff80001045bb54: ldr w0, [x2, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001045bb58: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.13 : ffff80001045bb5c: str w0, [x2, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045bb60: mov x3, #0xffffffffffffffff // #-1 : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff80001045bb64: mov x0, x1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.11 : ffff80001045bb68: mrs x1, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 1.17 : ffff80001045bb6c: add x0, x0, x1 2.24 : ffff80001045bb70: ldxr x5, [x0] 1.31 : ffff80001045bb74: add x5, x5, x3 0.08 : ffff80001045bb78: stxr w4, x5, [x0] 1.00 : ffff80001045bb7c: cbnz w4, ffff80001045bb70 : __read_once_size(): : __READ_ONCE_SIZE; 23.15 : ffff80001045bb80: ldr x0, [x2, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045bb84: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 22.28 : ffff80001045bb88: str w0, [x2, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045bb8c: cbnz x0, ffff80001045bba4 : percpu_ref_put_many(): 0.00 : ffff80001045bb90: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff80001045bb94: bl ffff800010160d20 <__rcu_read_unlock> : blk_queue_exit(): : percpu_ref_put(&q->q_usage_counter); : } 0.00 : ffff80001045bb98: ldr x19, [sp, #16] 0.00 : ffff80001045bb9c: ldp x29, x30, [sp], #32 0.00 : ffff80001045bba0: ret : __read_once_size(): : __READ_ONCE_SIZE; 9.01 : ffff80001045bba4: ldr x0, [x2, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045bba8: cbz x0, ffff80001045bb90 : rcu_read_unlock(): 32.51 : ffff80001045bbac: bl ffff800010160d20 <__rcu_read_unlock> : blk_queue_exit(): 1.30 : ffff80001045bbb0: ldr x19, [sp, #16] 2.29 : ffff80001045bbb4: ldp x29, x30, [sp], #32 0.00 : ffff80001045bbb8: ret : percpu_ref_put(&q->q_usage_counter); 0.00 : ffff80001045bbbc: add x0, x19, #0x5c0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001045bbc0: b ffff80001045bbe8 0.00 : ffff80001045bbc4: b ffff80001045bbe8 : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001045bbc8: mov x1, #0x1 // #1 0.00 : ffff80001045bbcc: neg x1, x1 0.00 : ffff80001045bbd0: ldaddal x1, x2, [x0] 0.00 : ffff80001045bbd4: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001045bbd8: cbnz x1, ffff80001045bb94 : ref->release(ref); 0.00 : ffff80001045bbdc: ldr x1, [x0, #16] 0.00 : ffff80001045bbe0: blr x1 0.00 : ffff80001045bbe4: b ffff80001045bb94 : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff80001045bbe8: mov x2, #0x1 // #1 0.00 : ffff80001045bbec: add x4, x19, #0x5c0 0.00 : ffff80001045bbf0: b ffff80001045ceb4 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001045bbf4: cbnz x1, ffff80001045bb94 0.00 : ffff80001045bbf8: b ffff80001045bbdc Percent | Source code & Disassembly of vmlinux for cycles (9551 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010117770 : : cpus_share_cache(): : rcu_read_unlock(); : } : : bool cpus_share_cache(int this_cpu, int that_cpu) : { : return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 0.22 : ffff800010117770: adrp x3, ffff800011909000 0.01 : ffff800010117774: add x3, x3, #0x928 0.00 : ffff800010117778: adrp x2, ffff80001151f000 0.00 : ffff80001011777c: add x2, x2, #0x124 0.15 : ffff800010117780: ldr x4, [x3, w0, sxtw #3] 1.33 : ffff800010117784: ldr x0, [x3, w1, sxtw #3] 13.74 : ffff800010117788: ldr w1, [x4, x2] 0.31 : ffff80001011778c: ldr w0, [x0, x2] 41.98 : ffff800010117790: cmp w1, w0 : } 41.51 : ffff800010117794: cset w0, eq // eq = none 0.74 : ffff800010117798: ret Percent | Source code & Disassembly of vmlinux for cycles (9044 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011c898 : : update_cfs_rq_h_load(): : */ : static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) : { : struct rq *rq = rq_of(cfs_rq); : struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; : unsigned long now = jiffies; 72.73 : ffff80001011c898: adrp x1, ffff800011907000 0.41 : ffff80001011c89c: ldr x4, [x1, #2432] : unsigned long load; : : if (cfs_rq->last_h_load_update == now) 0.08 : ffff80001011c8a0: ldr x1, [x0, #288] 0.00 : ffff80001011c8a4: cmp x1, x4 0.03 : ffff80001011c8a8: b.eq ffff80001011c934 // b.none : struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 0.19 : ffff80001011c8ac: ldr x2, [x0, #304] 0.13 : ffff80001011c8b0: ldr x1, [x0, #336] 0.04 : ffff80001011c8b4: ldrsw x2, [x2, #2568] 1.23 : ffff80001011c8b8: ldr x1, [x1, #240] 0.04 : ffff80001011c8bc: ldr x1, [x1, x2, lsl #3] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.28 : ffff80001011c8c0: str xzr, [x0, #296] : update_cfs_rq_h_load(): : return; : : WRITE_ONCE(cfs_rq->h_load_next, NULL); : for_each_sched_entity(se) { 0.00 : ffff80001011c8c4: cbz x1, ffff80001011c8fc : cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011c8c8: ldr x0, [x1, #128] : WRITE_ONCE(cfs_rq->h_load_next, se); : if (cfs_rq->last_h_load_update == now) 1.33 : ffff80001011c8cc: ldr x2, [x0, #288] : __write_once_size(): 0.11 : ffff80001011c8d0: str x1, [x0, #296] : update_cfs_rq_h_load(): 0.00 : ffff80001011c8d4: cmp x4, x2 0.00 : ffff80001011c8d8: b.ne ffff80001011c8f4 // b.any 0.00 : ffff80001011c8dc: b ffff80001011c904 : cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011c8e0: ldr x0, [x1, #128] : if (cfs_rq->last_h_load_update == now) 0.00 : ffff80001011c8e4: ldr x2, [x0, #288] : __write_once_size(): 0.00 : ffff80001011c8e8: str x1, [x0, #296] : update_cfs_rq_h_load(): 0.00 : ffff80001011c8ec: cmp x2, x4 0.00 : ffff80001011c8f0: b.eq ffff80001011c904 // b.none : for_each_sched_entity(se) { 0.01 : ffff80001011c8f4: ldr x1, [x1, #120] 0.00 : ffff80001011c8f8: cbnz x1, ffff80001011c8e0 : break; : } : : if (!se) { : cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 0.28 : ffff80001011c8fc: ldr x1, [x0, #160] : cfs_rq->last_h_load_update = now; 0.01 : ffff80001011c900: stp x1, x4, [x0, #280] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011c904: ldr x2, [x0, #296] : update_cfs_rq_h_load(): : } : : while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 0.00 : ffff80001011c908: cbz x2, ffff80001011c934 0.02 : ffff80001011c90c: ldr x1, [x0, #280] : load = cfs_rq->h_load; : load = div64_ul(load * se->avg.load_avg, 0.03 : ffff80001011c910: ldr x5, [x2, #224] 0.32 : ffff80001011c914: ldr x3, [x0, #160] : cfs_rq_load_avg(cfs_rq) + 1); : cfs_rq = group_cfs_rq(se); 0.00 : ffff80001011c918: ldr x0, [x2, #136] : load = div64_ul(load * se->avg.load_avg, 0.00 : ffff80001011c91c: add x3, x3, #0x1 0.00 : ffff80001011c920: mul x1, x1, x5 : __read_once_size(): 0.00 : ffff80001011c924: ldr x2, [x0, #296] : div64_u64(): : * : * Return: dividend / divisor : */ : static inline u64 div64_u64(u64 dividend, u64 divisor) : { : return dividend / divisor; 0.00 : ffff80001011c928: udiv x1, x1, x3 : update_cfs_rq_h_load(): : cfs_rq->h_load = load; : cfs_rq->last_h_load_update = now; 0.01 : ffff80001011c92c: stp x1, x4, [x0, #280] : while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 0.00 : ffff80001011c930: cbnz x2, ffff80001011c910 : } : } 22.69 : ffff80001011c934: ret Percent | Source code & Disassembly of vmlinux for cycles (8336 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101d7548 : : mempool_free(): : */ : void mempool_free(void *element, mempool_t *pool) : { : unsigned long flags; : : if (unlikely(element == NULL)) 1.41 : ffff8000101d7548: cbz x0, ffff8000101d758c : { 0.00 : ffff8000101d754c: stp x29, x30, [sp, #-32]! 0.00 : ffff8000101d7550: mov x29, sp 0.71 : ffff8000101d7554: stp x19, x20, [sp, #16] 0.00 : ffff8000101d7558: mov x19, x1 0.00 : ffff8000101d755c: mov x20, x0 : * performs "p = mempool_alloc(...);" while another task is doing : * "while (!p) cpu_relax(); mempool_free(p, ...);". This function : * may end up using curr_nr value which is from before allocation : * of @p without the following rmb. : */ : smp_rmb(); 3.68 : ffff8000101d7560: dmb ishld : * : * Waiters happen iff curr_nr is 0 and the above guarantee also : * ensures that there will be frees which return elements to the : * pool waking up the waiters. : */ : if (unlikely(pool->curr_nr < pool->min_nr)) { 88.79 : ffff8000101d7564: ldp w0, w1, [x1, #4] 0.00 : ffff8000101d7568: cmp w1, w0 0.00 : ffff8000101d756c: b.lt ffff8000101d7590 // b.tstop : wake_up(&pool->wait); : return; : } : spin_unlock_irqrestore(&pool->lock, flags); : } : pool->free(element, pool->pool_data); 0.81 : ffff8000101d7570: ldr x1, [x19, #24] 0.00 : ffff8000101d7574: mov x0, x20 0.35 : ffff8000101d7578: ldr x2, [x19, #40] 0.00 : ffff8000101d757c: blr x2 : } 3.99 : ffff8000101d7580: ldp x19, x20, [sp, #16] 0.25 : ffff8000101d7584: ldp x29, x30, [sp], #32 0.00 : ffff8000101d7588: ret 0.00 : ffff8000101d758c: ret : spin_lock_irqsave(&pool->lock, flags); 0.00 : ffff8000101d7590: mov x0, x19 0.00 : ffff8000101d7594: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : if (likely(pool->curr_nr < pool->min_nr)) { 0.00 : ffff8000101d7598: ldp w3, w2, [x19, #4] 0.00 : ffff8000101d759c: cmp w2, w3 0.00 : ffff8000101d75a0: b.ge ffff8000101d75d8 // b.tcont : add_element(): : pool->elements[pool->curr_nr++] = element; 0.00 : ffff8000101d75a4: ldr x3, [x19, #16] 0.00 : ffff8000101d75a8: add w1, w2, #0x1 0.00 : ffff8000101d75ac: str w1, [x19, #8] : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000101d75b0: mov x1, x0 0.00 : ffff8000101d75b4: mov x0, x19 : add_element(): 0.00 : ffff8000101d75b8: str x20, [x3, w2, sxtw #3] : spin_unlock_irqrestore(): 0.00 : ffff8000101d75bc: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : mempool_free(): : wake_up(&pool->wait); 0.00 : ffff8000101d75c0: mov x3, #0x0 // #0 0.00 : ffff8000101d75c4: mov w2, #0x1 // #1 0.00 : ffff8000101d75c8: mov w1, #0x3 // #3 0.00 : ffff8000101d75cc: add x0, x19, #0x30 0.00 : ffff8000101d75d0: bl ffff80001012f9c0 <__wake_up> : return; 0.00 : ffff8000101d75d4: b ffff8000101d7580 : spin_unlock_irqrestore(): 0.00 : ffff8000101d75d8: mov x1, x0 0.00 : ffff8000101d75dc: mov x0, x19 0.00 : ffff8000101d75e0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff8000101d75e4: b ffff8000101d7570 Percent | Source code & Disassembly of vmlinux for cycles (8270 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010456b18 : : bio_endio(): : * using bio_chain(). The ->bi_end_io() function will only be called the : * last time. At this point the BLK_TA_COMPLETE tracing event will be : * generated if BIO_TRACE_COMPLETION is set. : **/ : void bio_endio(struct bio *bio) : { 0.62 : ffff800010456b18: stp x29, x30, [sp, #-48]! 0.00 : ffff800010456b1c: mov x29, sp 1.46 : ffff800010456b20: stp x19, x20, [sp, #16] 0.00 : ffff800010456b24: mov x19, x0 2.66 : ffff800010456b28: str x21, [sp, #32] : * save/restore bi_end_io) - however, we want to avoid unbounded : * recursion and blowing the stack. Tail call optimization would : * handle this, but compiling with frame pointers also disables : * gcc's sibling call optimization. : */ : if (bio->bi_end_io == bio_chain_endio) { 0.00 : ffff800010456b2c: adrp x21, ffff800010456000 0.00 : ffff800010456b30: add x21, x21, #0xc50 : __ll_sc_atomic_sub_return(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010456b34: mov w20, #0x1 // #1 : bio_remaining_done(): : if (!bio_flagged(bio, BIO_CHAIN)) 2.11 : ffff800010456b38: ldrh w0, [x19, #20] 0.00 : ffff800010456b3c: tbz w0, #7, ffff800010456b78 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010456b40: ldr w0, [x19, #28] : bio_remaining_done(): : BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); 0.00 : ffff800010456b44: cmp w0, #0x0 0.00 : ffff800010456b48: b.le ffff800010456bfc : if (atomic_dec_and_test(&bio->__bi_remaining)) { 0.00 : ffff800010456b4c: add x0, x19, #0x1c : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010456b50: b ffff800010456be4 0.00 : ffff800010456b54: b ffff800010456be4 : __lse_atomic_sub_return(): : } : : ATOMIC_OP_SUB_RETURN(_relaxed, ) : ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC_OP_SUB_RETURN(_release, l, "memory") : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff800010456b58: mov w1, w20 0.00 : ffff800010456b5c: neg w1, w1 0.00 : ffff800010456b60: ldaddal w1, w2, [x0] 0.00 : ffff800010456b64: add w1, w1, w2 : bio_remaining_done(): 0.00 : ffff800010456b68: cbnz w1, ffff800010456bec : bio_clear_flag(): : bio->bi_flags |= (1U << bit); : } : : static inline void bio_clear_flag(struct bio *bio, unsigned int bit) : { : bio->bi_flags &= ~(1U << bit); 0.00 : ffff800010456b6c: ldrh w0, [x19, #20] 0.00 : ffff800010456b70: and w0, w0, #0xffffff7f 0.00 : ffff800010456b74: strh w0, [x19, #20] : bio_integrity(): : : #if defined(CONFIG_BLK_DEV_INTEGRITY) : : static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) : { : if (bio->bi_opf & REQ_INTEGRITY) 1.13 : ffff800010456b78: ldr w0, [x19, #16] 0.01 : ffff800010456b7c: tbz w0, #16, ffff800010456b98 : bio_integrity_endio(): : void blk_flush_integrity(void); : bool __bio_integrity_endio(struct bio *); : void bio_integrity_free(struct bio *bio); : static inline bool bio_integrity_endio(struct bio *bio) : { : if (bio_integrity(bio)) 0.00 : ffff800010456b80: ldr x0, [x19, #88] 0.00 : ffff800010456b84: cbz x0, ffff800010456b98 : return __bio_integrity_endio(bio); 0.00 : ffff800010456b88: mov x0, x19 0.00 : ffff800010456b8c: bl ffff800010480708 <__bio_integrity_endio> : bio_endio(): : if (!bio_integrity_endio(bio)) 0.00 : ffff800010456b90: tst w0, #0xff 0.00 : ffff800010456b94: b.eq ffff800010456bec // b.none : if (bio->bi_disk) 12.02 : ffff800010456b98: ldr x0, [x19, #8] 0.00 : ffff800010456b9c: cbz x0, ffff800010456bb4 : rq_qos_done_bio(bio->bi_disk->queue, bio); 4.51 : ffff800010456ba0: ldr x0, [x0, #1040] 9.91 : ffff800010456ba4: ldr x0, [x0, #24] : rq_qos_done_bio(): : __rq_qos_requeue(q->rq_qos, rq); : } : : static inline void rq_qos_done_bio(struct request_queue *q, struct bio *bio) : { : if (q->rq_qos) 0.01 : ffff800010456ba8: cbz x0, ffff800010456bb4 : __rq_qos_done_bio(q->rq_qos, bio); 0.00 : ffff800010456bac: mov x1, x19 0.00 : ffff800010456bb0: bl ffff800010476340 <__rq_qos_done_bio> : bio_endio(): : if (bio->bi_end_io == bio_chain_endio) { 48.69 : ffff800010456bb4: ldr x0, [x19, #56] 0.00 : ffff800010456bb8: cmp x0, x21 0.01 : ffff800010456bbc: b.ne ffff800010456c00 // b.any : __bio_chain_endio(): : struct bio *parent = bio->bi_private; 0.00 : ffff800010456bc0: ldr x1, [x19, #64] : if (!parent->bi_status) 0.00 : ffff800010456bc4: ldrb w0, [x1, #26] 0.00 : ffff800010456bc8: cbnz w0, ffff800010456bd4 : parent->bi_status = bio->bi_status; 0.00 : ffff800010456bcc: ldrb w0, [x19, #26] 0.00 : ffff800010456bd0: strb w0, [x1, #26] : bio_put(bio); 0.00 : ffff800010456bd4: mov x0, x19 : bio_endio(): : bio = __bio_chain_endio(bio); 0.00 : ffff800010456bd8: mov x19, x1 : __bio_chain_endio(): : bio_put(bio); 0.00 : ffff800010456bdc: bl ffff800010456ab8 : bio_endio(): : goto again; 0.00 : ffff800010456be0: b ffff800010456b38 : __ll_sc_atomic_sub_return(): 0.00 : ffff800010456be4: b ffff800010458b98 : bio_remaining_done(): : if (atomic_dec_and_test(&bio->__bi_remaining)) { 0.00 : ffff800010456be8: cbz w1, ffff800010456b6c : bio_endio(): : blk_throtl_bio_endio(bio); : /* release cgroup info */ : bio_uninit(bio); : if (bio->bi_end_io) : bio->bi_end_io(bio); : } 0.00 : ffff800010456bec: ldp x19, x20, [sp, #16] 0.00 : ffff800010456bf0: ldr x21, [sp, #32] 0.00 : ffff800010456bf4: ldp x29, x30, [sp], #48 0.00 : ffff800010456bf8: ret : bio_remaining_done(): : BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); 0.00 : ffff800010456bfc: brk #0x800 : bio_endio(): : if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) { 4.52 : ffff800010456c00: ldr x0, [x19, #8] 0.00 : ffff800010456c04: cbz x0, ffff800010456c24 : bio_flagged(): : return (bio->bi_flags & (1U << bit)) != 0; 0.77 : ffff800010456c08: ldrh w0, [x19, #20] : bio_endio(): 0.00 : ffff800010456c0c: tbz w0, #10, ffff800010456c24 : trace_block_bio_complete(bio->bi_disk->queue, bio, 0.00 : ffff800010456c10: ldrb w0, [x19, #26] 0.00 : ffff800010456c14: bl ffff800010459fd0 : bio_clear_flag(): : bio->bi_flags &= ~(1U << bit); 0.00 : ffff800010456c18: ldrh w0, [x19, #20] 0.00 : ffff800010456c1c: and w0, w0, #0xfffffbff 0.00 : ffff800010456c20: strh w0, [x19, #20] : bio_endio(): : bio_uninit(bio); 0.58 : ffff800010456c24: mov x0, x19 0.00 : ffff800010456c28: bl ffff800010456448 : if (bio->bi_end_io) 0.27 : ffff800010456c2c: ldr x1, [x19, #56] 0.00 : ffff800010456c30: cbz x1, ffff800010456bec : bio->bi_end_io(bio); 6.39 : ffff800010456c34: mov x0, x19 0.00 : ffff800010456c38: blr x1 : } 0.27 : ffff800010456c3c: ldp x19, x20, [sp, #16] 0.46 : ffff800010456c40: ldr x21, [sp, #32] 3.60 : ffff800010456c44: ldp x29, x30, [sp], #48 0.00 : ffff800010456c48: ret Percent | Source code & Disassembly of vmlinux for cycles (7967 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010458638 : : update_io_ticks(): : spin_unlock_irqrestore(&bio_dirty_lock, flags); : schedule_work(&bio_dirty_work); : } : : void update_io_ticks(struct hd_struct *part, unsigned long now) : { 3.37 : ffff800010458638: mov x3, x0 : unsigned long stamp; : again: : stamp = READ_ONCE(part->stamp); : if (unlikely(stamp != now)) { : if (likely(cmpxchg(&part->stamp, stamp, now) == stamp)) { : __part_stat_add(part, io_ticks, 1); 0.00 : ffff80001045863c: adrp x4, ffff80001151d000 0.00 : ffff800010458640: add x8, x4, #0x18 : { 0.00 : ffff800010458644: mov x5, x1 : __part_stat_add(part, io_ticks, 1); 1.25 : ffff800010458648: adrp x2, ffff800011909000 0.00 : ffff80001045864c: add x7, x2, #0x928 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.11 : ffff800010458650: ldr x4, [x3, #832] : update_io_ticks(): : if (unlikely(stamp != now)) { 0.00 : ffff800010458654: cmp x5, x4 0.00 : ffff800010458658: b.ne ffff800010458684 // b.any 16.41 : ffff80001045865c: nop : } : } : if (part->partno) { 65.91 : ffff800010458660: ldr w0, [x3, #820] 0.00 : ffff800010458664: cbz w0, ffff8000104586ac : part_to_disk(): : struct lockdep_map lockdep_map; : }; : : static inline struct gendisk *part_to_disk(struct hd_struct *part) : { : if (likely(part)) { 1.16 : ffff800010458668: cbz x3, ffff8000104586e4 : if (part->partno) : return dev_to_disk(part_to_dev(part)->parent); 0.97 : ffff80001045866c: ldr x3, [x3, #104] 0.00 : ffff800010458670: sub x3, x3, #0x70 : update_io_ticks(): : part = &part_to_disk(part)->part0; 0.00 : ffff800010458674: add x3, x3, #0x48 : __read_once_size(): 1.29 : ffff800010458678: ldr x4, [x3, #832] : update_io_ticks(): : if (unlikely(stamp != now)) { 0.00 : ffff80001045867c: cmp x5, x4 0.43 : ffff800010458680: b.eq ffff800010458660 // b.none : if (likely(cmpxchg(&part->stamp, stamp, now) == stamp)) { 0.01 : ffff800010458684: add x6, x3, #0x340 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010458688: b ffff8000104586b0 0.00 : ffff80001045868c: b ffff8000104586b0 : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff800010458690: mov x0, x6 0.00 : ffff800010458694: mov x1, x4 0.00 : ffff800010458698: mov x2, x5 0.00 : ffff80001045869c: mov x9, x1 0.00 : ffff8000104586a0: casal x9, x2, [x6] 0.56 : ffff8000104586a4: mov x0, x9 0.00 : ffff8000104586a8: b ffff8000104586b4 : update_io_ticks(): : goto again; : } : } 8.52 : ffff8000104586ac: ret : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff8000104586b0: b ffff800010458c20 : update_io_ticks(): : if (likely(cmpxchg(&part->stamp, stamp, now) == stamp)) { 0.00 : ffff8000104586b4: cmp x4, x0 0.00 : ffff8000104586b8: b.ne ffff800010458660 // b.any : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000104586bc: mrs x2, tpidr_el1 : update_io_ticks(): : __part_stat_add(part, io_ticks, 1); 0.00 : ffff8000104586c0: mov x1, x8 0.01 : ffff8000104586c4: ldr x0, [x3, #840] 0.01 : ffff8000104586c8: ldrsw x1, [x1, x2] 0.00 : ffff8000104586cc: ldr x1, [x7, x1, lsl #3] 0.00 : ffff8000104586d0: add x0, x0, x1 0.00 : ffff8000104586d4: ldr x1, [x0, #128] 0.00 : ffff8000104586d8: add x1, x1, #0x1 0.00 : ffff8000104586dc: str x1, [x0, #128] 0.00 : ffff8000104586e0: b ffff800010458660 : part_to_disk(): : else : return dev_to_disk(part_to_dev(part)); : } : return NULL; 0.00 : ffff8000104586e4: mov x3, #0x0 // #0 : update_io_ticks(): : part = &part_to_disk(part)->part0; 0.00 : ffff8000104586e8: add x3, x3, #0x48 0.00 : ffff8000104586ec: b ffff800010458678 Percent | Source code & Disassembly of vmlinux for cycles (7718 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011fbf8 : : reweight_entity(): : dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } : #endif : : static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, : unsigned long weight, unsigned long runnable) : { 0.01 : ffff80001011fbf8: stp x29, x30, [sp, #-48]! 0.04 : ffff80001011fbfc: mov x29, sp 2.13 : ffff80001011fc00: stp x19, x20, [sp, #16] 0.00 : ffff80001011fc04: mov x19, x1 0.40 : ffff80001011fc08: stp x21, x22, [sp, #32] 0.00 : ffff80001011fc0c: mov x20, x0 0.00 : ffff80001011fc10: mov x22, x2 0.00 : ffff80001011fc14: mov x21, x3 : if (se->on_rq) { 1.34 : ffff80001011fc18: ldr w1, [x1, #64] 0.00 : ffff80001011fc1c: cbz w1, ffff80001011fc80 : /* commit outstanding execution time */ : if (cfs_rq->curr == se) 0.00 : ffff80001011fc20: ldr x1, [x0, #72] 0.00 : ffff80001011fc24: cmp x1, x19 0.00 : ffff80001011fc28: b.eq ffff80001011fd70 // b.none : update_curr(cfs_rq); : account_entity_dequeue(cfs_rq, se); 0.00 : ffff80001011fc2c: mov x1, x19 0.00 : ffff80001011fc30: mov x0, x20 0.00 : ffff80001011fc34: bl ffff80001011c5c8 : dequeue_runnable_load_avg(): : cfs_rq->runnable_weight -= se->runnable_weight; 0.00 : ffff80001011fc38: ldr x2, [x19, #16] 0.00 : ffff80001011fc3c: ldr x0, [x20, #16] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011fc40: ldr x1, [x20, #168] : dequeue_runnable_load_avg(): 0.00 : ffff80001011fc44: sub x0, x0, x2 0.00 : ffff80001011fc48: str x0, [x20, #16] : sub_positive(&cfs_rq->avg.runnable_load_avg, se->avg.runnable_load_avg); 0.00 : ffff80001011fc4c: ldr x0, [x19, #232] 0.00 : ffff80001011fc50: sub x0, x1, x0 0.00 : ffff80001011fc54: cmp x1, x0 0.00 : ffff80001011fc58: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011fc5c: str x0, [x20, #168] : se_runnable(): : return scale_load_down(se->load.weight); : } : : static inline long se_runnable(struct sched_entity *se) : { : return scale_load_down(se->runnable_weight); 0.00 : ffff80001011fc60: ldr x0, [x19, #16] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011fc64: ldr x1, [x20, #144] : dequeue_runnable_load_avg(): : sub_positive(&cfs_rq->avg.runnable_load_sum, 0.00 : ffff80001011fc68: ldr x2, [x19, #208] : se_runnable(): 0.00 : ffff80001011fc6c: lsr x0, x0, #10 : dequeue_runnable_load_avg(): 0.00 : ffff80001011fc70: msub x0, x0, x2, x1 0.00 : ffff80001011fc74: cmp x1, x0 0.00 : ffff80001011fc78: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011fc7c: str x0, [x20, #144] : __read_once_size(): : __READ_ONCE_SIZE; 0.86 : ffff80001011fc80: ldr x2, [x20, #160] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff80001011fc84: lsr x1, x22, #10 : dequeue_load_avg(): : sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 2.33 : ffff80001011fc88: ldr x0, [x19, #224] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff80001011fc8c: lsr x4, x21, #10 : dequeue_load_avg(): 0.00 : ffff80001011fc90: sub x0, x2, x0 0.00 : ffff80001011fc94: cmp x2, x0 0.61 : ffff80001011fc98: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.91 : ffff80001011fc9c: str x0, [x20, #160] : se_weight(): : return scale_load_down(se->load.weight); 0.23 : ffff80001011fca0: ldr x0, [x19] : __read_once_size(): : __READ_ONCE_SIZE; 47.15 : ffff80001011fca4: ldr x2, [x20, #136] : dequeue_load_avg(): : sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 0.16 : ffff80001011fca8: ldr x3, [x19, #200] : se_weight(): 0.00 : ffff80001011fcac: lsr x0, x0, #10 : dequeue_load_avg(): 0.00 : ffff80001011fcb0: msub x0, x0, x3, x2 0.00 : ffff80001011fcb4: cmp x2, x0 1.40 : ffff80001011fcb8: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.05 : ffff80001011fcbc: str x0, [x20, #136] : div_u64_rem(): : * This is commonly provided by 32bit archs to provide an optimized 64bit : * divide. : */ : static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) : { : *remainder = dividend % divisor; 0.00 : ffff80001011fcc0: mov w0, #0xb67e // #46718 : update_load_set(): : lw->weight = w; 2.11 : ffff80001011fcc4: str x22, [x19] : reweight_entity(): : se->runnable_weight = runnable; : update_load_set(&se->load, weight); : : #ifdef CONFIG_SMP : do { : u32 divider = LOAD_AVG_MAX - 1024 + se->avg.period_contrib; 0.16 : ffff80001011fcc8: ldr w2, [x19, #220] : update_load_set(): : lw->inv_weight = 0; 1.23 : ffff80001011fccc: str wzr, [x19, #8] : div_u64_rem(): 0.00 : ffff80001011fcd0: add w2, w2, w0 : reweight_entity(): : se->runnable_weight = runnable; 0.22 : ffff80001011fcd4: str x21, [x19, #16] : : se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); : se->avg.runnable_load_avg = : div_u64(se_runnable(se) * se->avg.runnable_load_sum, divider); 1.97 : ffff80001011fcd8: ldp x0, x3, [x19, #200] : se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 0.00 : ffff80001011fcdc: mul x0, x1, x0 : div_u64(se_runnable(se) * se->avg.runnable_load_sum, divider); 0.00 : ffff80001011fce0: mul x1, x4, x3 : div_u64_rem(): : return dividend / divisor; 0.00 : ffff80001011fce4: udiv x0, x0, x2 3.77 : ffff80001011fce8: udiv x1, x1, x2 : reweight_entity(): : se->avg.runnable_load_avg = 0.59 : ffff80001011fcec: stp x0, x1, [x19, #224] : enqueue_load_avg(): : cfs_rq->avg.load_avg += se->avg.load_avg; 0.12 : ffff80001011fcf0: ldr x1, [x20, #160] : cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 2.76 : ffff80001011fcf4: ldr x2, [x20, #136] : cfs_rq->avg.load_avg += se->avg.load_avg; 0.18 : ffff80001011fcf8: add x0, x1, x0 19.71 : ffff80001011fcfc: str x0, [x20, #160] : se_weight(): 0.99 : ffff80001011fd00: ldr x0, [x19] : enqueue_load_avg(): : cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 4.61 : ffff80001011fd04: ldr x1, [x19, #200] : se_weight(): 0.00 : ffff80001011fd08: lsr x0, x0, #10 : enqueue_load_avg(): 0.00 : ffff80001011fd0c: madd x0, x0, x1, x2 0.04 : ffff80001011fd10: str x0, [x20, #136] : reweight_entity(): : } while (0); : #endif : : enqueue_load_avg(cfs_rq, se); : if (se->on_rq) { 1.87 : ffff80001011fd14: ldr w0, [x19, #64] 0.00 : ffff80001011fd18: cbz w0, ffff80001011fd60 : account_entity_enqueue(cfs_rq, se); 0.00 : ffff80001011fd1c: mov x1, x19 0.00 : ffff80001011fd20: mov x0, x20 0.00 : ffff80001011fd24: bl ffff80001011c540 : enqueue_runnable_load_avg(): : cfs_rq->runnable_weight += se->runnable_weight; 0.00 : ffff80001011fd28: ldr x0, [x19, #16] 0.00 : ffff80001011fd2c: ldr x2, [x20, #16] : cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg; 0.00 : ffff80001011fd30: ldr x1, [x20, #168] : cfs_rq->runnable_weight += se->runnable_weight; 0.00 : ffff80001011fd34: add x2, x2, x0 0.00 : ffff80001011fd38: str x2, [x20, #16] : cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum; 0.00 : ffff80001011fd3c: ldr x3, [x20, #144] : cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg; 0.00 : ffff80001011fd40: ldr x0, [x19, #232] 0.00 : ffff80001011fd44: add x1, x1, x0 0.00 : ffff80001011fd48: str x1, [x20, #168] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff80001011fd4c: ldr x0, [x19, #16] : enqueue_runnable_load_avg(): : cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum; 0.00 : ffff80001011fd50: ldr x1, [x19, #208] : se_runnable(): 0.00 : ffff80001011fd54: lsr x0, x0, #10 : enqueue_runnable_load_avg(): 0.00 : ffff80001011fd58: madd x0, x0, x1, x3 0.00 : ffff80001011fd5c: str x0, [x20, #144] : reweight_entity(): : enqueue_runnable_load_avg(cfs_rq, se); : } : } 0.59 : ffff80001011fd60: ldp x19, x20, [sp, #16] 1.18 : ffff80001011fd64: ldp x21, x22, [sp, #32] 0.10 : ffff80001011fd68: ldp x29, x30, [sp], #48 0.17 : ffff80001011fd6c: ret : update_curr(cfs_rq); 0.00 : ffff80001011fd70: bl ffff80001011fab8 0.00 : ffff80001011fd74: b ffff80001011fc2c Percent | Source code & Disassembly of vmlinux for cycles (7622 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010455200 : : bio_advance(): : * be updated on the last bvec as well. : * : * @bio will then represent the remaining, uncompleted portion of the io. : */ : void bio_advance(struct bio *bio, unsigned bytes) : { 0.86 : ffff800010455200: stp x29, x30, [sp, #-48]! 0.00 : ffff800010455204: mov x29, sp 11.29 : ffff800010455208: stp x19, x20, [sp, #16] 0.00 : ffff80001045520c: mov x20, x0 1.45 : ffff800010455210: str x21, [sp, #32] 0.04 : ffff800010455214: mov w19, w1 : if (bio_integrity(bio)) 3.55 : ffff800010455218: ldr w2, [x0, #16] : bio_integrity(): : : #if defined(CONFIG_BLK_DEV_INTEGRITY) : : static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) : { : if (bio->bi_opf & REQ_INTEGRITY) 0.00 : ffff80001045521c: tbz w2, #16, ffff800010455230 : bio_advance(): 0.00 : ffff800010455220: ldr x3, [x0, #88] 0.00 : ffff800010455224: cbz x3, ffff800010455230 : bio_integrity_advance(bio, bytes); 0.00 : ffff800010455228: bl ffff8000104807a8 0.00 : ffff80001045522c: ldr w2, [x20, #16] : bio_advance_iter(): : iter->bi_sector += bytes >> 9; 1.17 : ffff800010455230: ldr x3, [x20, #32] : bio_no_advance_iter(): : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff800010455234: and w0, w2, #0xff : bio_advance_iter(): : iter->bi_sector += bytes >> 9; 0.00 : ffff800010455238: add x21, x20, #0x20 0.00 : ffff80001045523c: lsr w1, w19, #9 2.93 : ffff800010455240: add x1, x3, x1 : bio_no_advance_iter(): : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff800010455244: sub w0, w0, #0x5 : bio_advance_iter(): : iter->bi_sector += bytes >> 9; 0.13 : ffff800010455248: str x1, [x20, #32] : bio_no_advance_iter(): : bio_op(bio) == REQ_OP_WRITE_SAME || 0.00 : ffff80001045524c: tst w0, #0xfffffffb : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff800010455250: mov w1, #0xfb // #251 0.00 : ffff800010455254: and w2, w2, w1 : bio_op(bio) == REQ_OP_WRITE_SAME || 2.60 : ffff800010455258: ccmp w2, #0x3, #0x4, ne // ne = any 0.34 : ffff80001045525c: ldr w1, [x21, #8] 0.00 : ffff800010455260: b.ne ffff80001045527c // b.any : bio_advance_iter(): : iter->bi_size -= bytes; 0.00 : ffff800010455264: sub w1, w1, w19 0.00 : ffff800010455268: str w1, [x21, #8] : bio_advance(): : : bio_advance_iter(bio, &bio->bi_iter, bytes); : } 0.00 : ffff80001045526c: ldp x19, x20, [sp, #16] 0.00 : ffff800010455270: ldr x21, [sp, #32] 0.00 : ffff800010455274: ldp x29, x30, [sp], #48 0.00 : ffff800010455278: ret : bvec_iter_advance(): : static inline bool bvec_iter_advance(const struct bio_vec *bv, : struct bvec_iter *iter, unsigned bytes) : { : unsigned int idx = iter->bi_idx; : : if (WARN_ONCE(bytes > iter->bi_size, 4.11 : ffff80001045527c: cmp w19, w1 : unsigned int idx = iter->bi_idx; 8.99 : ffff800010455280: ldr w2, [x21, #12] : bio_advance_iter(): : bvec_iter_advance(bio->bi_io_vec, iter, bytes); 4.98 : ffff800010455284: ldr x3, [x20, #104] : bvec_iter_advance(): : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff800010455288: b.hi ffff8000104552d8 // b.pmore : "Attempted to advance past end of bvec iter\n")) { : iter->bi_size = 0; : return false; : } : : iter->bi_size -= bytes; 16.63 : ffff80001045528c: sub w1, w1, w19 : bytes += iter->bi_bvec_done; 0.98 : ffff800010455290: ldr w0, [x21, #16] : iter->bi_size -= bytes; 1.57 : ffff800010455294: str w1, [x21, #8] : : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010455298: adds w1, w19, w0 0.00 : ffff80001045529c: b.ne ffff8000104552b0 // b.any 0.00 : ffff8000104552a0: b ffff8000104552c4 18.80 : ffff8000104552a4: subs w1, w1, w0 : bytes -= bv[idx].bv_len; : idx++; 0.00 : ffff8000104552a8: add w2, w2, #0x1 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff8000104552ac: b.eq ffff8000104552c4 // b.none 1.95 : ffff8000104552b0: ubfiz x0, x2, #4, #32 0.00 : ffff8000104552b4: add x0, x3, x0 1.37 : ffff8000104552b8: ldr w0, [x0, #8] 0.00 : ffff8000104552bc: cmp w0, w1 0.01 : ffff8000104552c0: b.ls ffff8000104552a4 // b.plast : } : : iter->bi_idx = idx; : iter->bi_bvec_done = bytes; 5.45 : ffff8000104552c4: stp w2, w1, [x21, #12] : bio_advance(): 5.49 : ffff8000104552c8: ldp x19, x20, [sp, #16] 3.24 : ffff8000104552cc: ldr x21, [sp, #32] 2.05 : ffff8000104552d0: ldp x29, x30, [sp], #48 0.00 : ffff8000104552d4: ret : bvec_iter_advance(): : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff8000104552d8: adrp x1, ffff800011ad8000 0.00 : ffff8000104552dc: ldrb w0, [x1, #1289] 0.00 : ffff8000104552e0: cbz w0, ffff8000104552ec : iter->bi_size = 0; 0.00 : ffff8000104552e4: str wzr, [x21, #8] 0.00 : ffff8000104552e8: b ffff80001045526c : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff8000104552ec: mov w2, #0x1 // #1 0.00 : ffff8000104552f0: adrp x0, ffff8000111e9000 0.00 : ffff8000104552f4: strb w2, [x1, #1289] 0.00 : ffff8000104552f8: add x0, x0, #0x680 0.00 : ffff8000104552fc: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010455300: brk #0x800 : iter->bi_size = 0; 0.00 : ffff800010455304: str wzr, [x21, #8] 0.00 : ffff800010455308: b ffff80001045526c Percent | Source code & Disassembly of vmlinux for cycles (8117 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001014c358 <__irq_wake_thread>: : __irq_wake_thread(): : printk(KERN_WARNING "IRQ %d device %s returned IRQ_WAKE_THREAD " : "but no thread function available.", irq, action->name); : } : : void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action) : { 0.06 : ffff80001014c358: stp x29, x30, [sp, #-48]! 0.30 : ffff80001014c35c: mov x29, sp 1.20 : ffff80001014c360: str x19, [sp, #16] 0.24 : ffff80001014c364: adrp x19, ffff800011909000 0.10 : ffff80001014c368: add x2, x19, #0x908 0.75 : ffff80001014c36c: ldr x3, [x2] 2.42 : ffff80001014c370: str x3, [x29, #40] 0.00 : ffff80001014c374: mov x3, #0x0 // #0 : /* : * In case the thread crashed and was killed we just pretend that : * we handled the interrupt. The hardirq handler has disabled the : * device interrupt, so no irq storm is lurking. : */ : if (action->thread->flags & PF_EXITING) 0.56 : ffff80001014c378: ldr x2, [x1, #40] 0.11 : ffff80001014c37c: ldr w2, [x2, #44] 0.00 : ffff80001014c380: tbz w2, #2, ffff80001014c3a4 <__irq_wake_thread+0x4c> : * like the finalize_oneshot() code. See comment above. : */ : atomic_inc(&desc->threads_active); : : wake_up_process(action->thread); : } 2.60 : ffff80001014c384: add x19, x19, #0x908 16.51 : ffff80001014c388: ldr x1, [x29, #40] 21.84 : ffff80001014c38c: ldr x0, [x19] 0.00 : ffff80001014c390: eor x0, x1, x0 0.00 : ffff80001014c394: cbnz x0, ffff80001014c40c <__irq_wake_thread+0xb4> 7.59 : ffff80001014c398: ldr x19, [sp, #16] 11.22 : ffff80001014c39c: ldp x29, x30, [sp], #48 0.00 : ffff80001014c3a0: ret : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.95 : ffff80001014c3a4: ldr x2, [x1, #64] : test_and_set_bit(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (READ_ONCE(*p) & mask) 0.00 : ffff80001014c3a8: tbnz w2, #0, ffff80001014c384 <__irq_wake_thread+0x2c> : __irq_wake_thread(): : if (test_and_set_bit(IRQTF_RUNTHREAD, &action->thread_flags)) 0.43 : ffff80001014c3ac: add x3, x1, #0x40 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001014c3b0: b ffff80001014c3f4 <__irq_wake_thread+0x9c> 0.00 : ffff80001014c3b4: b ffff80001014c3f4 <__irq_wake_thread+0x9c> : __lse_atomic64_fetch_or(): : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff80001014c3b8: mov x2, #0x1 // #1 0.10 : ffff80001014c3bc: ldsetal x2, x2, [x3] : __irq_wake_thread(): 15.89 : ffff80001014c3c0: tbnz w2, #0, ffff80001014c384 <__irq_wake_thread+0x2c> : desc->threads_oneshot |= action->thread_mask; 0.26 : ffff80001014c3c4: ldr x2, [x0, #248] : atomic_inc(&desc->threads_active); 0.00 : ffff80001014c3c8: add x4, x0, #0x100 : desc->threads_oneshot |= action->thread_mask; 6.81 : ffff80001014c3cc: ldr x3, [x1, #72] 0.00 : ffff80001014c3d0: orr x2, x2, x3 7.08 : ffff80001014c3d4: str x2, [x0, #248] : arch_static_branch_jump(): 0.00 : ffff80001014c3d8: b ffff80001014c400 <__irq_wake_thread+0xa8> 0.03 : ffff80001014c3dc: b ffff80001014c400 <__irq_wake_thread+0xa8> : __lse_atomic_add(): : ATOMIC_OP(add, stadd) 0.00 : ffff80001014c3e0: mov w2, #0x1 // #1 0.01 : ffff80001014c3e4: stadd w2, [x4] : __irq_wake_thread(): : wake_up_process(action->thread); 2.93 : ffff80001014c3e8: ldr x0, [x1, #40] 0.00 : ffff80001014c3ec: bl ffff800010116f00 0.00 : ffff80001014c3f0: b ffff80001014c384 <__irq_wake_thread+0x2c> : __ll_sc_atomic64_fetch_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff80001014c3f4: add x5, x1, #0x40 0.00 : ffff80001014c3f8: b ffff80001014c6a4 0.00 : ffff80001014c3fc: b ffff80001014c3c0 <__irq_wake_thread+0x68> : __ll_sc_atomic_add(): : ATOMIC_OPS(add, add, I) 0.00 : ffff80001014c400: add x0, x0, #0x100 0.00 : ffff80001014c404: b ffff80001014c6c0 0.00 : ffff80001014c408: b ffff80001014c3e8 <__irq_wake_thread+0x90> : __irq_wake_thread(): : } 0.00 : ffff80001014c40c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (8008 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001048d988 <_find_next_bit.constprop.1>: : _find_next_bit(): : const unsigned long *addr2, unsigned long nbits, : unsigned long start, unsigned long invert, unsigned long le) : { : unsigned long tmp, mask; : : if (unlikely(start >= nbits)) 1.04 : ffff80001048d988: cmp x3, x2 0.28 : ffff80001048d98c: b.cs ffff80001048d9e8 <_find_next_bit.constprop.1+0x60> // b.hs, b.nlast : return nbits; : : tmp = addr1[start / BITS_PER_LONG]; 4.53 : ffff80001048d990: lsr x6, x3, #6 2.37 : ffff80001048d994: ldr x5, [x0, x6, lsl #3] : if (addr2) 3.03 : ffff80001048d998: cbz x1, ffff80001048d9a4 <_find_next_bit.constprop.1+0x1c> : tmp &= addr2[start / BITS_PER_LONG]; 0.14 : ffff80001048d99c: ldr x6, [x1, x6, lsl #3] 0.01 : ffff80001048d9a0: and x5, x5, x6 : tmp ^= invert; 33.58 : ffff80001048d9a4: eor x6, x5, x4 : : /* Handle 1st word. */ : mask = BITMAP_FIRST_WORD_MASK(start); 0.00 : ffff80001048d9a8: mov x5, #0xffffffffffffffff // #-1 0.04 : ffff80001048d9ac: lsl x5, x5, x3 : if (le) : mask = swab(mask); : : tmp &= mask; : : start = round_down(start, BITS_PER_LONG); 0.00 : ffff80001048d9b0: and x3, x3, #0xffffffffffffffc0 : : while (!tmp) { 0.09 : ffff80001048d9b4: ands x5, x5, x6 0.05 : ffff80001048d9b8: b.eq ffff80001048d9dc <_find_next_bit.constprop.1+0x54> // b.none 0.05 : ffff80001048d9bc: b ffff80001048d9f4 <_find_next_bit.constprop.1+0x6c> : start += BITS_PER_LONG; : if (start >= nbits) : return nbits; : : tmp = addr1[start / BITS_PER_LONG]; 11.24 : ffff80001048d9c0: lsr x6, x3, #6 0.51 : ffff80001048d9c4: ldr x5, [x0, x6, lsl #3] : if (addr2) 0.16 : ffff80001048d9c8: cbz x1, ffff80001048d9d4 <_find_next_bit.constprop.1+0x4c> : tmp &= addr2[start / BITS_PER_LONG]; 0.05 : ffff80001048d9cc: ldr x6, [x1, x6, lsl #3] 0.00 : ffff80001048d9d0: and x5, x5, x6 : while (!tmp) { 18.20 : ffff80001048d9d4: cmp x4, x5 0.00 : ffff80001048d9d8: b.ne ffff80001048d9f0 <_find_next_bit.constprop.1+0x68> // b.any : start += BITS_PER_LONG; 13.56 : ffff80001048d9dc: add x3, x3, #0x40 : if (start >= nbits) 0.26 : ffff80001048d9e0: cmp x2, x3 0.36 : ffff80001048d9e4: b.hi ffff80001048d9c0 <_find_next_bit.constprop.1+0x38> // b.pmore : : if (le) : tmp = swab(tmp); : : return min(start + __ffs(tmp), nbits); : } 3.76 : ffff80001048d9e8: mov x0, x2 0.00 : ffff80001048d9ec: ret : tmp ^= invert; 0.01 : ffff80001048d9f0: eor x5, x4, x5 : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 6.56 : ffff80001048d9f4: rbit x5, x5 0.01 : ffff80001048d9f8: clz x5, x5 : _find_next_bit(): : return min(start + __ffs(tmp), nbits); 0.00 : ffff80001048d9fc: add x3, x5, x3 0.03 : ffff80001048da00: cmp x2, x3 0.08 : ffff80001048da04: csel x2, x2, x3, ls // ls = plast : } 0.00 : ffff80001048da08: mov x0, x2 0.00 : ffff80001048da0c: ret Percent | Source code & Disassembly of vmlinux for cycles (6950 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010720478 : : arm_smmu_tlb_inv_range(): : } : : static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, : size_t granule, bool leaf, : struct arm_smmu_domain *smmu_domain) : { 1.94 : ffff800010720478: sub sp, sp, #0x490 3.55 : ffff80001072047c: stp x29, x30, [sp] 0.04 : ffff800010720480: mov x29, sp 0.12 : ffff800010720484: str x21, [sp, #32] 0.00 : ffff800010720488: adrp x21, ffff800011909000 0.01 : ffff80001072048c: str x25, [sp, #64] 0.00 : ffff800010720490: add x5, x21, #0x908 1.65 : ffff800010720494: str x28, [sp, #88] 0.00 : ffff800010720498: mov x25, x4 : u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS]; : struct arm_smmu_device *smmu = smmu_domain->smmu; : unsigned long start = iova, end = iova + size; : int i = 0; : struct arm_smmu_cmdq_ent cmd = { 3.09 : ffff80001072049c: stp xzr, xzr, [x29, #104] : { 0.04 : ffff8000107204a0: ldr x4, [x5] 9.87 : ffff8000107204a4: str x4, [x29, #1160] 0.00 : ffff8000107204a8: mov x4, #0x0 // #0 : struct arm_smmu_cmdq_ent cmd = { 0.99 : ffff8000107204ac: strb w3, [x29, #116] 2.48 : ffff8000107204b0: stp xzr, xzr, [x29, #120] : struct arm_smmu_device *smmu = smmu_domain->smmu; 0.16 : ffff8000107204b4: ldr x28, [x25] : .tlbi = { : .leaf = leaf, : }, : }; : : if (!size) 0.00 : ffff8000107204b8: cbz x1, ffff800010720598 1.85 : ffff8000107204bc: str x19, [x29, #16] 0.85 : ffff8000107204c0: stp x22, x23, [x29, #40] 2.69 : ffff8000107204c4: str x24, [x29, #56] 0.00 : ffff8000107204c8: mov x24, x2 0.20 : ffff8000107204cc: stp x26, x27, [x29, #72] 0.00 : ffff8000107204d0: mov x27, x0 0.00 : ffff8000107204d4: mov x26, x1 : return; : : if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 1.70 : ffff8000107204d8: ldr w0, [x25, #56] 0.00 : ffff8000107204dc: cbz w0, ffff8000107205c4 : cmd.opcode = CMDQ_OP_TLBI_NH_VA; : cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; : } else { : cmd.opcode = CMDQ_OP_TLBI_S2_IPA; : cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; 0.00 : ffff8000107204e0: ldrh w0, [x25, #64] : cmd.opcode = CMDQ_OP_TLBI_S2_IPA; 0.00 : ffff8000107204e4: mov w1, #0x2a // #42 0.00 : ffff8000107204e8: strb w1, [x29, #104] : cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; 0.00 : ffff8000107204ec: strh w0, [x29, #114] : unsigned long start = iova, end = iova + size; 1.90 : ffff8000107204f0: add x23, x27, x26 : } : : while (iova < end) { 0.00 : ffff8000107204f4: mov w19, #0x0 // #0 0.00 : ffff8000107204f8: add x22, x29, #0x88 0.00 : ffff8000107204fc: cmp x27, x23 0.74 : ffff800010720500: b.cs ffff800010720564 // b.hs, b.nlast 2.64 : ffff800010720504: str x20, [x29, #24] 0.00 : ffff800010720508: add x22, x29, #0x88 0.00 : ffff80001072050c: mov x20, x27 0.00 : ffff800010720510: mov w19, #0x0 // #0 0.09 : ffff800010720514: nop : arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, false); : i = 0; : } : : cmd.tlbi.addr = iova; : arm_smmu_cmdq_build_cmd(&cmds[i * CMDQ_ENT_DWORDS], &cmd); 1.90 : ffff800010720518: lsl w0, w19, #1 : cmd.tlbi.addr = iova; 0.84 : ffff80001072051c: str x20, [x29, #120] : arm_smmu_cmdq_build_cmd(&cmds[i * CMDQ_ENT_DWORDS], &cmd); 7.41 : ffff800010720520: add x1, x29, #0x68 : iova += granule; 0.04 : ffff800010720524: add x20, x20, x24 : arm_smmu_cmdq_build_cmd(&cmds[i * CMDQ_ENT_DWORDS], &cmd); 0.27 : ffff800010720528: add x0, x22, w0, sxtw #3 20.69 : ffff80001072052c: bl ffff80001071d5e0 : i++; 0.22 : ffff800010720530: add w19, w19, #0x1 : while (iova < end) { 0.00 : ffff800010720534: cmp x23, x20 0.00 : ffff800010720538: b.ls ffff800010720560 // b.plast : if (i == CMDQ_BATCH_ENTRIES) { 0.00 : ffff80001072053c: cmp w19, #0x40 0.00 : ffff800010720540: b.ne ffff800010720518 // b.any : arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, false); 0.00 : ffff800010720544: mov w2, w19 0.00 : ffff800010720548: mov w3, #0x0 // #0 0.00 : ffff80001072054c: mov x1, x22 0.00 : ffff800010720550: mov x0, x28 : i = 0; 0.00 : ffff800010720554: mov w19, #0x0 // #0 : arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, false); 0.00 : ffff800010720558: bl ffff80001071e6f8 0.00 : ffff80001072055c: b ffff800010720518 0.30 : ffff800010720560: ldr x20, [x29, #24] : } : : arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, true); 0.00 : ffff800010720564: mov w3, #0x1 // #1 0.00 : ffff800010720568: mov w2, w19 0.01 : ffff80001072056c: mov x1, x22 6.93 : ffff800010720570: mov x0, x28 0.00 : ffff800010720574: bl ffff80001071e6f8 : : /* : * Unfortunately, this can't be leaf-only since we may have : * zapped an entire table. : */ : arm_smmu_atc_inv_domain(smmu_domain, 0, start, size); 0.03 : ffff800010720578: mov x2, x26 0.00 : ffff80001072057c: mov x1, x27 0.44 : ffff800010720580: mov x0, x25 0.00 : ffff800010720584: bl ffff80001071f1c8 2.34 : ffff800010720588: ldr x19, [x29, #16] 0.46 : ffff80001072058c: ldp x22, x23, [x29, #40] 8.80 : ffff800010720590: ldr x24, [x29, #56] 0.01 : ffff800010720594: ldp x26, x27, [x29, #72] : } 0.00 : ffff800010720598: add x21, x21, #0x908 2.70 : ffff80001072059c: ldr x1, [x29, #1160] 0.01 : ffff8000107205a0: ldr x0, [x21] 0.00 : ffff8000107205a4: eor x0, x1, x0 0.00 : ffff8000107205a8: cbnz x0, ffff8000107205d8 5.78 : ffff8000107205ac: ldp x29, x30, [sp] 0.12 : ffff8000107205b0: ldr x21, [sp, #32] 0.30 : ffff8000107205b4: ldr x25, [sp, #64] 0.00 : ffff8000107205b8: ldr x28, [sp, #88] 0.00 : ffff8000107205bc: add sp, sp, #0x490 0.00 : ffff8000107205c0: ret : cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; 0.83 : ffff8000107205c4: ldrh w0, [x25, #96] : cmd.opcode = CMDQ_OP_TLBI_NH_VA; 0.00 : ffff8000107205c8: mov w1, #0x12 // #18 2.96 : ffff8000107205cc: strb w1, [x29, #104] : cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; 0.00 : ffff8000107205d0: strh w0, [x29, #112] 0.00 : ffff8000107205d4: b ffff8000107204f0 0.00 : ffff8000107205d8: stp x19, x20, [x29, #16] 0.00 : ffff8000107205dc: stp x22, x23, [x29, #40] 0.00 : ffff8000107205e0: str x24, [x29, #56] 0.00 : ffff8000107205e4: stp x26, x27, [x29, #72] : } 0.00 : ffff8000107205e8: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (6589 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107d3dd0 : : nvme_pci_complete_rq(): : nvme_cleanup_cmd(req); : return ret; : } : : static void nvme_pci_complete_rq(struct request *req) : { 4.39 : ffff8000107d3dd0: stp x29, x30, [sp, #-32]! 0.01 : ffff8000107d3dd4: mov x29, sp 0.74 : ffff8000107d3dd8: stp x19, x20, [sp, #16] 0.03 : ffff8000107d3ddc: mov x19, x0 : struct nvme_iod *iod = blk_mq_rq_to_pdu(req); : struct nvme_dev *dev = iod->nvmeq->dev; : : if (blk_integrity_rq(req)) 0.33 : ffff8000107d3de0: ldr w3, [x0, #24] : struct nvme_dev *dev = iod->nvmeq->dev; 3.89 : ffff8000107d3de4: ldr x0, [x0, #312] 59.36 : ffff8000107d3de8: ldr x20, [x0] : if (blk_integrity_rq(req)) 0.49 : ffff8000107d3dec: tbnz w3, #16, ffff8000107d3e20 : blk_rq_nr_phys_segments(): : * own special payload. In that case we still return 1 here so that this : * special payload will be mapped. : */ : static inline unsigned short blk_rq_nr_phys_segments(struct request *rq) : { : if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 11.33 : ffff8000107d3df0: ldr w0, [x19, #28] 0.00 : ffff8000107d3df4: tbnz w0, #18, ffff8000107d3e00 : nvme_pci_complete_rq(): : dma_unmap_page(dev->dev, iod->meta_dma, : rq_integrity_vec(req)->bv_len, rq_data_dir(req)); : if (blk_rq_nr_phys_segments(req)) 0.27 : ffff8000107d3df8: ldrh w0, [x19, #194] 0.43 : ffff8000107d3dfc: cbz w0, ffff8000107d3e0c : nvme_unmap_data(dev, req); 8.52 : ffff8000107d3e00: mov x1, x19 0.02 : ffff8000107d3e04: mov x0, x20 0.06 : ffff8000107d3e08: bl ffff8000107d35e0 : nvme_complete_rq(req); 7.26 : ffff8000107d3e0c: mov x0, x19 0.00 : ffff8000107d3e10: bl ffff8000107cfe70 : } 0.14 : ffff8000107d3e14: ldp x19, x20, [sp, #16] 2.74 : ffff8000107d3e18: ldp x29, x30, [sp], #32 0.00 : ffff8000107d3e1c: ret : rq_integrity_vec(): : * Return the first bvec that contains integrity data. Only drivers that are : * limited to a single integrity segment should use this helper. : */ : static inline struct bio_vec *rq_integrity_vec(struct request *rq) : { : if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1)) 0.00 : ffff8000107d3e20: ldr x2, [x19] : nvme_pci_complete_rq(): : dma_unmap_page(dev->dev, iod->meta_dma, 0.00 : ffff8000107d3e24: ldr x1, [x19, #352] 0.00 : ffff8000107d3e28: ldr x0, [x20, #336] : rq_integrity_vec(): 0.00 : ffff8000107d3e2c: ldrh w2, [x2, #1130] 0.00 : ffff8000107d3e30: cmp w2, #0x1 0.00 : ffff8000107d3e34: b.hi ffff8000107d3e74 // b.pmore : return NULL; : return rq->bio->bi_integrity->bip_vec; 0.00 : ffff8000107d3e38: ldr x2, [x19, #56] 0.00 : ffff8000107d3e3c: ldr x2, [x2, #88] 0.00 : ffff8000107d3e40: ldr x2, [x2, #96] : dma_unmap_page_attrs(): : } : : static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, : size_t size, enum dma_data_direction dir, unsigned long attrs) : { : const struct dma_map_ops *ops = get_dma_ops(dev); 0.00 : ffff8000107d3e44: ldr x4, [x0, #576] 0.00 : ffff8000107d3e48: and w3, w3, #0x1 : nvme_pci_complete_rq(): 0.00 : ffff8000107d3e4c: ldr w2, [x2, #8] : get_dma_ops(): : if (dev->dma_ops) 0.00 : ffff8000107d3e50: cbz x4, ffff8000107d3e68 : dma_unmap_page_attrs(): : : BUG_ON(!valid_dma_direction(dir)); : if (dma_is_direct(ops)) : dma_direct_unmap_page(dev, addr, size, dir, attrs); : else if (ops->unmap_page) 0.00 : ffff8000107d3e54: ldr x5, [x4, #40] 0.00 : ffff8000107d3e58: cbz x5, ffff8000107d3df0 : ops->unmap_page(dev, addr, size, dir, attrs); 0.00 : ffff8000107d3e5c: mov x4, #0x0 // #0 0.00 : ffff8000107d3e60: blr x5 0.00 : ffff8000107d3e64: b ffff8000107d3df0 : dma_direct_unmap_page(dev, addr, size, dir, attrs); 0.00 : ffff8000107d3e68: mov x4, #0x0 // #0 0.00 : ffff8000107d3e6c: bl ffff800010163b00 0.00 : ffff8000107d3e70: b ffff8000107d3df0 : rq_integrity_vec(): : if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1)) 0.00 : ffff8000107d3e74: brk #0x800 : return NULL; 0.00 : ffff8000107d3e78: mov x2, #0x0 // #0 0.00 : ffff8000107d3e7c: ldr w3, [x19, #24] 0.00 : ffff8000107d3e80: b ffff8000107d3e44 Percent | Source code & Disassembly of vmlinux for cycles (6159 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107117a8 <__iommu_unmap>: : __iommu_unmap(): : EXPORT_SYMBOL_GPL(iommu_map_atomic); : : static size_t __iommu_unmap(struct iommu_domain *domain, : unsigned long iova, size_t size, : struct iommu_iotlb_gather *iotlb_gather) : { 11.81 : ffff8000107117a8: stp x29, x30, [sp, #-64]! 0.23 : ffff8000107117ac: mov x29, sp 4.54 : ffff8000107117b0: stp x20, x21, [sp, #24] 0.00 : ffff8000107117b4: mov x21, x0 2.34 : ffff8000107117b8: str x23, [sp, #48] : const struct iommu_ops *ops = domain->ops; 4.28 : ffff8000107117bc: ldr x23, [x0, #8] : size_t unmapped_page, unmapped = 0; : unsigned long orig_iova = iova; : unsigned int min_pagesz; : : if (unlikely(ops->unmap == NULL || 0.85 : ffff8000107117c0: ldr x0, [x23, #48] 5.92 : ffff8000107117c4: cbz x0, ffff80001071187c <__iommu_unmap+0xd4> 9.47 : ffff8000107117c8: ldr x0, [x21, #16] 0.26 : ffff8000107117cc: cbz x0, ffff80001071187c <__iommu_unmap+0xd4> 3.83 : ffff8000107117d0: str x24, [x29, #56] 0.10 : ffff8000107117d4: mov x24, x3 : domain->pgsize_bitmap == 0UL)) : return 0; : : if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 1.10 : ffff8000107117d8: ldr w3, [x21] 0.00 : ffff8000107117dc: tbz w3, #0, ffff800010711894 <__iommu_unmap+0xec> : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 0.46 : ffff8000107117e0: rbit x4, x0 : __iommu_unmap(): : return 0; : : /* find out the minimum page size supported */ : min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 0.00 : ffff8000107117e4: mov w3, #0x1 // #1 : __ffs(): 0.00 : ffff8000107117e8: clz x4, x4 1.14 : ffff8000107117ec: str x19, [x29, #16] 4.73 : ffff8000107117f0: str x22, [x29, #40] : __iommu_unmap(): : /* : * The virtual address, as well as the size of the mapping, must be : * aligned (at least) to the size of the smallest page supported : * by the hardware : */ : if (!IS_ALIGNED(iova | size, min_pagesz)) { 0.45 : ffff8000107117f4: orr x5, x1, x2 : min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 0.00 : ffff8000107117f8: lsl w4, w3, w4 0.00 : ffff8000107117fc: mov x3, x4 : if (!IS_ALIGNED(iova | size, min_pagesz)) { 2.88 : ffff800010711800: sub x4, x4, #0x1 0.08 : ffff800010711804: mov x19, x1 0.00 : ffff800010711808: mov x22, x2 0.03 : ffff80001071180c: tst x5, x4 0.24 : ffff800010711810: b.ne ffff8000107118a0 <__iommu_unmap+0xf8> // b.any : size_t unmapped_page, unmapped = 0; 0.80 : ffff800010711814: mov x20, #0x0 // #0 : : /* : * Keep iterating until we either unmap 'size' bytes (or more) : * or we hit an area that isn't mapped. : */ : while (unmapped < size) { 0.00 : ffff800010711818: cbnz x2, ffff800010711834 <__iommu_unmap+0x8c> 0.00 : ffff80001071181c: b ffff80001071185c <__iommu_unmap+0xb4> : : pr_debug("unmapped: iova 0x%lx size 0x%zx\n", : iova, unmapped_page); : : iova += unmapped_page; : unmapped += unmapped_page; 2.97 : ffff800010711820: add x20, x20, x0 : iova += unmapped_page; 0.00 : ffff800010711824: add x19, x19, x0 : while (unmapped < size) { 0.00 : ffff800010711828: cmp x22, x20 0.00 : ffff80001071182c: b.ls ffff80001071185c <__iommu_unmap+0xb4> // b.plast 0.00 : ffff800010711830: ldr x0, [x21, #16] : size_t pgsize = iommu_pgsize(domain, iova, size - unmapped); 8.05 : ffff800010711834: sub x2, x22, x20 0.02 : ffff800010711838: mov x1, x19 0.03 : ffff80001071183c: bl ffff800010711750 : unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather); 0.83 : ffff800010711840: mov x2, x0 0.31 : ffff800010711844: ldr x4, [x23, #48] 0.00 : ffff800010711848: mov x1, x19 0.33 : ffff80001071184c: mov x3, x24 0.00 : ffff800010711850: mov x0, x21 1.72 : ffff800010711854: blr x4 : if (!unmapped_page) 6.65 : ffff800010711858: cbnz x0, ffff800010711820 <__iommu_unmap+0x78> 0.02 : ffff80001071185c: ldr x19, [x29, #16] 10.78 : ffff800010711860: ldr x22, [x29, #40] 2.53 : ffff800010711864: ldr x24, [x29, #56] : } : : trace_unmap(orig_iova, size, unmapped); : return unmapped; : } 0.00 : ffff800010711868: mov x0, x20 5.72 : ffff80001071186c: ldr x23, [sp, #48] 0.15 : ffff800010711870: ldp x20, x21, [sp, #24] 4.35 : ffff800010711874: ldp x29, x30, [sp], #64 0.00 : ffff800010711878: ret : return 0; 0.00 : ffff80001071187c: mov x20, #0x0 // #0 : } 0.00 : ffff800010711880: ldr x23, [sp, #48] 0.00 : ffff800010711884: mov x0, x20 0.00 : ffff800010711888: ldp x20, x21, [sp, #24] 0.00 : ffff80001071188c: ldp x29, x30, [sp], #64 0.00 : ffff800010711890: ret : return 0; 0.00 : ffff800010711894: mov x20, #0x0 // #0 0.00 : ffff800010711898: ldr x24, [x29, #56] 0.00 : ffff80001071189c: b ffff800010711868 <__iommu_unmap+0xc0> : pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", 0.00 : ffff8000107118a0: adrp x0, ffff800011269000 : return 0; 0.00 : ffff8000107118a4: mov x20, #0x0 // #0 : pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", 0.00 : ffff8000107118a8: add x0, x0, #0x870 0.00 : ffff8000107118ac: bl ffff80001014a85c : return 0; 0.00 : ffff8000107118b0: ldr x19, [x29, #16] 0.00 : ffff8000107118b4: ldr x22, [x29, #40] 0.00 : ffff8000107118b8: ldr x24, [x29, #56] 0.00 : ffff8000107118bc: b ffff800010711868 <__iommu_unmap+0xc0> Percent | Source code & Disassembly of vmlinux for cycles (6011 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010154c60 : : irq_find_mapping(): : * @domain: domain owning this hardware interrupt : * @hwirq: hardware irq number in that domain space : */ : unsigned int irq_find_mapping(struct irq_domain *domain, : irq_hw_number_t hwirq) : { 0.00 : ffff800010154c60: stp x29, x30, [sp, #-48]! 0.00 : ffff800010154c64: mov x29, sp 0.00 : ffff800010154c68: stp x19, x20, [sp, #16] 0.00 : ffff800010154c6c: mov x20, x1 5.32 : ffff800010154c70: str x21, [sp, #32] 0.00 : ffff800010154c74: mov x19, x0 : struct irq_data *data; : : /* Look for default domain if nececssary */ : if (domain == NULL) 0.00 : ffff800010154c78: cbz x0, ffff800010154d28 : domain = irq_default_domain; : if (domain == NULL) : return 0; : : if (hwirq < domain->revmap_direct_max_irq) { 1.15 : ffff800010154c7c: ldr w0, [x19, #88] 0.00 : ffff800010154c80: cmp x0, x20 0.00 : ffff800010154c84: b.hi ffff800010154ce4 // b.pmore : if (data && data->hwirq == hwirq) : return hwirq; : } : : /* Check if the hwirq is in the linear revmap. */ : if (hwirq < domain->revmap_size) 0.29 : ffff800010154c88: ldr w0, [x19, #92] 0.00 : ffff800010154c8c: cmp x0, x20 0.00 : ffff800010154c90: b.ls ffff800010154cb0 // b.plast : return domain->linear_revmap[hwirq]; 0.00 : ffff800010154c94: add x20, x20, #0x24 0.00 : ffff800010154c98: ldr w21, [x19, x20, lsl #2] : : rcu_read_lock(); : data = radix_tree_lookup(&domain->revmap_tree, hwirq); : rcu_read_unlock(); : return data ? data->irq : 0; : } 0.00 : ffff800010154c9c: mov w0, w21 0.00 : ffff800010154ca0: ldr x21, [sp, #32] 0.00 : ffff800010154ca4: ldp x19, x20, [sp, #16] 0.00 : ffff800010154ca8: ldp x29, x30, [sp], #48 0.00 : ffff800010154cac: ret : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.49 : ffff800010154cb0: bl ffff80001015c1b0 <__rcu_read_lock> : irq_find_mapping(): : data = radix_tree_lookup(&domain->revmap_tree, hwirq); 0.62 : ffff800010154cb4: mov x1, x20 0.00 : ffff800010154cb8: add x0, x19, #0x60 0.00 : ffff800010154cbc: bl ffff800010cc3e60 44.47 : ffff800010154cc0: mov x19, x0 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff800010154cc4: bl ffff800010160d20 <__rcu_read_unlock> : irq_find_mapping(): : return data ? data->irq : 0; 1.61 : ffff800010154cc8: cbz x19, ffff800010154d34 1.56 : ffff800010154ccc: ldr w21, [x19, #4] : } 44.46 : ffff800010154cd0: ldp x19, x20, [sp, #16] 0.00 : ffff800010154cd4: mov w0, w21 0.00 : ffff800010154cd8: ldr x21, [sp, #32] 0.02 : ffff800010154cdc: ldp x29, x30, [sp], #48 0.00 : ffff800010154ce0: ret : irq_domain_get_irq_data(): : struct irq_data *irq_domain_get_irq_data(struct irq_domain *domain, : unsigned int virq) : { : struct irq_data *irq_data; : : for (irq_data = irq_get_irq_data(virq); irq_data; 0.00 : ffff800010154ce4: mov w0, w20 : irq_find_mapping(): : data = irq_domain_get_irq_data(domain, hwirq); 0.00 : ffff800010154ce8: mov w21, w20 : irq_domain_get_irq_data(): : for (irq_data = irq_get_irq_data(virq); irq_data; 0.00 : ffff800010154cec: bl ffff800010151170 0.00 : ffff800010154cf0: cbnz x0, ffff800010154d00 0.00 : ffff800010154cf4: b ffff800010154c88 : irq_data = irq_data->parent_data) 0.00 : ffff800010154cf8: ldr x0, [x0, #40] : for (irq_data = irq_get_irq_data(virq); irq_data; 0.00 : ffff800010154cfc: cbz x0, ffff800010154c88 : if (irq_data->domain == domain) 0.00 : ffff800010154d00: ldr x1, [x0, #32] 0.00 : ffff800010154d04: cmp x1, x19 0.00 : ffff800010154d08: b.ne ffff800010154cf8 // b.any : irq_find_mapping(): : if (data && data->hwirq == hwirq) 0.00 : ffff800010154d0c: ldr x0, [x0, #8] 0.00 : ffff800010154d10: cmp x0, x20 0.00 : ffff800010154d14: b.eq ffff800010154c9c // b.none : if (hwirq < domain->revmap_size) 0.00 : ffff800010154d18: ldr w0, [x19, #92] 0.00 : ffff800010154d1c: cmp x0, x20 0.00 : ffff800010154d20: b.hi ffff800010154c94 // b.pmore 0.00 : ffff800010154d24: b ffff800010154cb0 : domain = irq_default_domain; 0.00 : ffff800010154d28: adrp x0, ffff800011b20000 <__log_buf+0x1fc10> 0.00 : ffff800010154d2c: ldr x19, [x0, #3232] : if (domain == NULL) 0.00 : ffff800010154d30: cbnz x19, ffff800010154c7c : return 0; 0.00 : ffff800010154d34: mov w21, #0x0 // #0 : } 0.00 : ffff800010154d38: mov w0, w21 0.00 : ffff800010154d3c: ldp x19, x20, [sp, #16] 0.00 : ffff800010154d40: ldr x21, [sp, #32] 0.00 : ffff800010154d44: ldp x29, x30, [sp], #48 0.00 : ffff800010154d48: ret Percent | Source code & Disassembly of vmlinux for cycles (5802 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cb7dc0 <__memcpy>: : __efistub_memcpy(): : C_l .req x11 : C_h .req x12 : D_l .req x13 : D_h .req x14 : : mov dst, dstin 3.38 : ffff800010cb7dc0: mov x6, x0 : cmp count, #16 0.00 : ffff800010cb7dc4: cmp x2, #0x10 : /*When memory length is less than 16, the accessed are not aligned.*/ : b.lo .Ltiny15 0.03 : ffff800010cb7dc8: b.cc ffff800010cb7e40 <__memcpy+0x80> // b.lo, b.ul, b.last : : neg tmp2, src 0.43 : ffff800010cb7dcc: neg x4, x1 : ands tmp2, tmp2, #15/* Bytes to reach alignment. */ 0.00 : ffff800010cb7dd0: ands x4, x4, #0xf : b.eq .LSrcAligned 0.00 : ffff800010cb7dd4: b.eq ffff800010cb7e0c <__memcpy+0x4c> // b.none : sub count, count, tmp2 0.00 : ffff800010cb7dd8: sub x2, x2, x4 : * Copy the leading memory data from src to dst in an increasing : * address order.By this way,the risk of overwriting the source : * memory data is eliminated when the distance between src and : * dst is less than 16. The memory accesses here are alignment. : */ : tbz tmp2, #0, 1f 0.00 : ffff800010cb7ddc: tbz w4, #0, ffff800010cb7de8 <__memcpy+0x28> : ldrb1 tmp1w, src, #1 0.00 : ffff800010cb7de0: ldrb w3, [x1], #1 : strb1 tmp1w, dst, #1 0.00 : ffff800010cb7de4: strb w3, [x6], #1 : 1: : tbz tmp2, #1, 2f 0.00 : ffff800010cb7de8: tbz w4, #1, ffff800010cb7df4 <__memcpy+0x34> : ldrh1 tmp1w, src, #2 0.00 : ffff800010cb7dec: ldrh w3, [x1], #2 : strh1 tmp1w, dst, #2 0.00 : ffff800010cb7df0: strh w3, [x6], #2 : 2: : tbz tmp2, #2, 3f 0.00 : ffff800010cb7df4: tbz w4, #2, ffff800010cb7e00 <__memcpy+0x40> : ldr1 tmp1w, src, #4 0.00 : ffff800010cb7df8: ldr w3, [x1], #4 : str1 tmp1w, dst, #4 0.00 : ffff800010cb7dfc: str w3, [x6], #4 : 3: : tbz tmp2, #3, .LSrcAligned 0.00 : ffff800010cb7e00: tbz w4, #3, ffff800010cb7e0c <__memcpy+0x4c> : ldr1 tmp1, src, #8 0.00 : ffff800010cb7e04: ldr x3, [x1], #8 : str1 tmp1, dst, #8 0.00 : ffff800010cb7e08: str x3, [x6], #8 : : .LSrcAligned: : cmp count, #64 2.80 : ffff800010cb7e0c: cmp x2, #0x40 : b.ge .Lcpy_over64 0.05 : ffff800010cb7e10: b.ge ffff800010cb7e74 <__memcpy+0xb4> // b.tcont : .Ltail63: : /* : * Copy up to 48 bytes of data. At this point we only need the : * bottom 6 bits of count to be accurate. : */ : ands tmp1, count, #0x30 0.32 : ffff800010cb7e14: ands x3, x2, #0x30 : b.eq .Ltiny15 0.00 : ffff800010cb7e18: b.eq ffff800010cb7e40 <__memcpy+0x80> // b.none : cmp tmp1w, #0x20 0.00 : ffff800010cb7e1c: cmp w3, #0x20 : b.eq 1f 0.00 : ffff800010cb7e20: b.eq ffff800010cb7e30 <__memcpy+0x70> // b.none : b.lt 2f 0.00 : ffff800010cb7e24: b.lt ffff800010cb7e38 <__memcpy+0x78> // b.tstop : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb7e28: ldp x7, x8, [x1], #16 : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb7e2c: stp x7, x8, [x6], #16 : 1: : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb7e30: ldp x7, x8, [x1], #16 : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb7e34: stp x7, x8, [x6], #16 : 2: : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb7e38: ldp x7, x8, [x1], #16 : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb7e3c: stp x7, x8, [x6], #16 : * precondition that src address is at least 16 bytes bigger than dst : * address,otherwise some source data will be overwritten when memove : * call memcpy directly. To make memmove simpler and decouple the : * memcpy's dependency on memmove, withdrew the original process. : */ : tbz count, #3, 1f 0.00 : ffff800010cb7e40: tbz w2, #3, ffff800010cb7e4c <__memcpy+0x8c> : ldr1 tmp1, src, #8 0.00 : ffff800010cb7e44: ldr x3, [x1], #8 : str1 tmp1, dst, #8 0.00 : ffff800010cb7e48: str x3, [x6], #8 : 1: : tbz count, #2, 2f 0.00 : ffff800010cb7e4c: tbz w2, #2, ffff800010cb7e58 <__memcpy+0x98> : ldr1 tmp1w, src, #4 0.00 : ffff800010cb7e50: ldr w3, [x1], #4 : str1 tmp1w, dst, #4 0.00 : ffff800010cb7e54: str w3, [x6], #4 : 2: : tbz count, #1, 3f 0.00 : ffff800010cb7e58: tbz w2, #1, ffff800010cb7e64 <__memcpy+0xa4> : ldrh1 tmp1w, src, #2 0.00 : ffff800010cb7e5c: ldrh w3, [x1], #2 : strh1 tmp1w, dst, #2 0.00 : ffff800010cb7e60: strh w3, [x6], #2 : 3: : tbz count, #0, .Lexitfunc 0.00 : ffff800010cb7e64: tbz w2, #0, ffff800010cb7f10 <__memcpy+0x150> : ldrb1 tmp1w, src, #1 0.00 : ffff800010cb7e68: ldrb w3, [x1], #1 : strb1 tmp1w, dst, #1 0.00 : ffff800010cb7e6c: strb w3, [x6], #1 : : b .Lexitfunc 0.00 : ffff800010cb7e70: b ffff800010cb7f10 <__memcpy+0x150> : : .Lcpy_over64: : subs count, count, #128 1.30 : ffff800010cb7e74: subs x2, x2, #0x80 : b.ge .Lcpy_body_large 0.00 : ffff800010cb7e78: b.ge ffff800010cb7ec0 <__memcpy+0x100> // b.tcont : /* : * Less than 128 bytes to copy, so handle 64 here and then jump : * to the tail. : */ : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb7e7c: ldp x7, x8, [x1], #16 : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb7e80: stp x7, x8, [x6], #16 : ldp1 B_l, B_h, src, #16 0.00 : ffff800010cb7e84: ldp x9, x10, [x1], #16 : ldp1 C_l, C_h, src, #16 0.00 : ffff800010cb7e88: ldp x11, x12, [x1], #16 : stp1 B_l, B_h, dst, #16 0.00 : ffff800010cb7e8c: stp x9, x10, [x6], #16 : stp1 C_l, C_h, dst, #16 0.00 : ffff800010cb7e90: stp x11, x12, [x6], #16 : ldp1 D_l, D_h, src, #16 0.00 : ffff800010cb7e94: ldp x13, x14, [x1], #16 : stp1 D_l, D_h, dst, #16 0.00 : ffff800010cb7e98: stp x13, x14, [x6], #16 : : tst count, #0x3f 0.00 : ffff800010cb7e9c: tst x2, #0x3f : b.ne .Ltail63 0.00 : ffff800010cb7ea0: b.ne ffff800010cb7e14 <__memcpy+0x54> // b.any : b .Lexitfunc 0.00 : ffff800010cb7ea4: b ffff800010cb7f10 <__memcpy+0x150> 0.00 : ffff800010cb7ea8: nop 0.00 : ffff800010cb7eac: nop 0.00 : ffff800010cb7eb0: nop 0.00 : ffff800010cb7eb4: nop 0.00 : ffff800010cb7eb8: nop 0.00 : ffff800010cb7ebc: nop : * 64 bytes per line this ensures the entire loop is in one line. : */ : .p2align L1_CACHE_SHIFT : .Lcpy_body_large: : /* pre-get 64 bytes data. */ : ldp1 A_l, A_h, src, #16 11.02 : ffff800010cb7ec0: ldp x7, x8, [x1], #16 : ldp1 B_l, B_h, src, #16 29.76 : ffff800010cb7ec4: ldp x9, x10, [x1], #16 : ldp1 C_l, C_h, src, #16 3.92 : ffff800010cb7ec8: ldp x11, x12, [x1], #16 : ldp1 D_l, D_h, src, #16 1.27 : ffff800010cb7ecc: ldp x13, x14, [x1], #16 : 1: : /* : * interlace the load of next 64 bytes data block with store of the last : * loaded 64 bytes data. : */ : stp1 A_l, A_h, dst, #16 1.99 : ffff800010cb7ed0: stp x7, x8, [x6], #16 : ldp1 A_l, A_h, src, #16 4.73 : ffff800010cb7ed4: ldp x7, x8, [x1], #16 : stp1 B_l, B_h, dst, #16 11.84 : ffff800010cb7ed8: stp x9, x10, [x6], #16 : ldp1 B_l, B_h, src, #16 2.38 : ffff800010cb7edc: ldp x9, x10, [x1], #16 : stp1 C_l, C_h, dst, #16 1.49 : ffff800010cb7ee0: stp x11, x12, [x6], #16 : ldp1 C_l, C_h, src, #16 1.87 : ffff800010cb7ee4: ldp x11, x12, [x1], #16 : stp1 D_l, D_h, dst, #16 4.00 : ffff800010cb7ee8: stp x13, x14, [x6], #16 : ldp1 D_l, D_h, src, #16 1.97 : ffff800010cb7eec: ldp x13, x14, [x1], #16 : subs count, count, #64 0.00 : ffff800010cb7ef0: subs x2, x2, #0x40 : b.ge 1b 0.00 : ffff800010cb7ef4: b.ge ffff800010cb7ed0 <__memcpy+0x110> // b.tcont : stp1 A_l, A_h, dst, #16 2.20 : ffff800010cb7ef8: stp x7, x8, [x6], #16 : stp1 B_l, B_h, dst, #16 1.54 : ffff800010cb7efc: stp x9, x10, [x6], #16 : stp1 C_l, C_h, dst, #16 4.12 : ffff800010cb7f00: stp x11, x12, [x6], #16 : stp1 D_l, D_h, dst, #16 2.51 : ffff800010cb7f04: stp x13, x14, [x6], #16 : : tst count, #0x3f 0.00 : ffff800010cb7f08: tst x2, #0x3f : b.ne .Ltail63 0.00 : ffff800010cb7f0c: b.ne ffff800010cb7e14 <__memcpy+0x54> // b.any : : .weak memcpy : SYM_FUNC_START_ALIAS(__memcpy) : SYM_FUNC_START_PI(memcpy) : #include "copy_template.S" : ret 5.07 : ffff800010cb7f10: ret : ... Percent | Source code & Disassembly of vmlinux for cycles (6063 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010150cf8 : : irq_chip_eoi_parent(): : /** : * irq_chip_eoi_parent - Invoke EOI on the parent interrupt : * @data: Pointer to interrupt specific data : */ : void irq_chip_eoi_parent(struct irq_data *data) : { 22.82 : ffff800010150cf8: stp x29, x30, [sp, #-16]! 0.00 : ffff800010150cfc: mov x29, sp : data = data->parent_data; 9.01 : ffff800010150d00: ldr x1, [x0, #40] : data->chip->irq_eoi(data); 0.02 : ffff800010150d04: mov x0, x1 2.32 : ffff800010150d08: ldr x1, [x1, #24] 57.53 : ffff800010150d0c: ldr x1, [x1, #80] 0.00 : ffff800010150d10: blr x1 : } 8.30 : ffff800010150d14: ldp x29, x30, [sp], #16 0.00 : ffff800010150d18: ret Percent | Source code & Disassembly of vmlinux for cycles (5666 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001071d188 : : arm_smmu_tlb_inv_page_nosync(): : } : : static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, : unsigned long iova, size_t granule, : void *cookie) : { 5.32 : ffff80001071d188: stp x29, x30, [sp, #-48]! 0.00 : ffff80001071d18c: mov x29, sp 0.16 : ffff80001071d190: stp x19, x20, [sp, #16] 0.00 : ffff80001071d194: mov x19, x0 0.04 : ffff80001071d198: stp x21, x22, [sp, #32] 0.00 : ffff80001071d19c: mov x21, x1 : iommu_iotlb_gather_add_page(): : : static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, : struct iommu_iotlb_gather *gather, : unsigned long iova, size_t size) : { : unsigned long start = iova, end = start + size; 0.00 : ffff80001071d1a0: add x22, x1, x2 : arm_smmu_tlb_inv_page_nosync(): 0.00 : ffff80001071d1a4: mov x20, x2 : iommu_iotlb_gather_add_page(): : /* : * If the new page is disjoint from the current range or is mapped at : * a different granularity, then sync the TLB so that the gather : * structure can be rewritten. : */ : if (gather->pgsize != size || 0.41 : ffff80001071d1a8: ldr x1, [x0, #16] 0.00 : ffff80001071d1ac: cmp x2, x1 0.00 : ffff80001071d1b0: b.eq ffff80001071d21c // b.none : end < gather->start || start > gather->end) { : if (gather->pgsize) 5.45 : ffff80001071d1b4: cbz x1, ffff80001071d214 : iommu_tlb_sync(): : if (domain->ops->iotlb_sync) 0.00 : ffff80001071d1b8: add x0, x3, #0x88 0.00 : ffff80001071d1bc: ldr x1, [x0, #8] 0.00 : ffff80001071d1c0: ldr x2, [x1, #72] 0.00 : ffff80001071d1c4: cbz x2, ffff80001071d1d0 : domain->ops->iotlb_sync(domain, iotlb_gather); 0.00 : ffff80001071d1c8: mov x1, x19 0.00 : ffff80001071d1cc: blr x2 : iommu_iotlb_gather_init(): : *gather = (struct iommu_iotlb_gather) { 0.00 : ffff80001071d1d0: mov x1, #0xffffffffffffffff // #-1 0.00 : ffff80001071d1d4: stp xzr, xzr, [x19] 0.00 : ffff80001071d1d8: mov x0, x1 0.00 : ffff80001071d1dc: mov x2, #0x0 // #0 0.00 : ffff80001071d1e0: str x1, [x19] 0.00 : ffff80001071d1e4: str xzr, [x19, #16] : iommu_iotlb_gather_add_page(): : iommu_tlb_sync(domain, gather); : gather->pgsize = size; 0.11 : ffff80001071d1e8: str x20, [x19, #16] : } : : if (gather->end < end) 0.00 : ffff80001071d1ec: cmp x22, x2 0.00 : ffff80001071d1f0: b.ls ffff80001071d1f8 // b.plast : gather->end = end; 0.49 : ffff80001071d1f4: str x22, [x19, #8] : : if (gather->start > start) 0.00 : ffff80001071d1f8: cmp x21, x0 0.00 : ffff80001071d1fc: b.cs ffff80001071d204 // b.hs, b.nlast : gather->start = start; 5.41 : ffff80001071d200: str x21, [x19] : arm_smmu_tlb_inv_page_nosync(): : struct arm_smmu_domain *smmu_domain = cookie; : struct iommu_domain *domain = &smmu_domain->domain; : : iommu_iotlb_gather_add_page(domain, gather, iova, granule); : } 2.01 : ffff80001071d204: ldp x19, x20, [sp, #16] 39.80 : ffff80001071d208: ldp x21, x22, [sp, #32] 40.65 : ffff80001071d20c: ldp x29, x30, [sp], #48 0.00 : ffff80001071d210: ret 0.14 : ffff80001071d214: ldp x0, x2, [x19] 0.00 : ffff80001071d218: b ffff80001071d1e8 : iommu_iotlb_gather_add_page(): : end < gather->start || start > gather->end) { 0.00 : ffff80001071d21c: ldr x0, [x0] : if (gather->pgsize != size || 0.00 : ffff80001071d220: cmp x22, x0 0.00 : ffff80001071d224: b.cc ffff80001071d1b4 // b.lo, b.ul, b.last : end < gather->start || start > gather->end) { 0.00 : ffff80001071d228: ldr x2, [x19, #8] 0.00 : ffff80001071d22c: cmp x21, x2 0.00 : ffff80001071d230: b.ls ffff80001071d1ec // b.plast : if (gather->pgsize) 0.00 : ffff80001071d234: cbnz x1, ffff80001071d1b8 0.00 : ffff80001071d238: b ffff80001071d214 Percent | Source code & Disassembly of vmlinux for cycles (5682 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011fab8 : : update_curr(): : : /* : * Update the current task's runtime statistics. : */ : static void update_curr(struct cfs_rq *cfs_rq) : { 6.57 : ffff80001011fab8: stp x29, x30, [sp, #-48]! 0.00 : ffff80001011fabc: mov x29, sp 0.18 : ffff80001011fac0: str x19, [sp, #16] : struct sched_entity *curr = cfs_rq->curr; 0.11 : ffff80001011fac4: ldr x19, [x0, #72] : u64 now = rq_clock_task(rq_of(cfs_rq)); 1.88 : ffff80001011fac8: ldr x1, [x0, #304] 6.97 : ffff80001011facc: ldr x1, [x1, #2432] : u64 delta_exec; : : if (unlikely(!curr)) 0.00 : ffff80001011fad0: cbz x19, ffff80001011fb28 0.22 : ffff80001011fad4: str x20, [x29, #24] : return; : : delta_exec = now - curr->exec_start; 0.00 : ffff80001011fad8: ldr x20, [x19, #72] 0.00 : ffff80001011fadc: sub x20, x1, x20 : if (unlikely((s64)delta_exec <= 0)) 0.02 : ffff80001011fae0: cmp x20, #0x0 0.00 : ffff80001011fae4: b.le ffff80001011fb24 0.04 : ffff80001011fae8: mov x7, x0 : curr->exec_start = now; : : schedstat_set(curr->statistics.exec_max, : max(delta_exec, curr->statistics.exec_max)); : : curr->sum_exec_runtime += delta_exec; 0.02 : ffff80001011faec: ldr x0, [x19, #80] : calc_delta_fair(): : if (unlikely(se->load.weight != NICE_0_LOAD)) 0.00 : ffff80001011faf0: ldr x3, [x19] 0.00 : ffff80001011faf4: mov x2, x20 : update_curr(): : curr->sum_exec_runtime += delta_exec; 0.00 : ffff80001011faf8: add x0, x0, x20 0.04 : ffff80001011fafc: stp x1, x0, [x19, #72] : calc_delta_fair(): : if (unlikely(se->load.weight != NICE_0_LOAD)) 0.00 : ffff80001011fb00: cmp x3, #0x100, lsl #12 0.00 : ffff80001011fb04: b.ne ffff80001011fbb0 // b.any : update_curr(): : schedstat_add(cfs_rq->exec_clock, delta_exec); : : curr->vruntime += calc_delta_fair(delta_exec, curr); 0.00 : ffff80001011fb08: ldr x1, [x19, #88] : update_min_vruntime(cfs_rq); 0.00 : ffff80001011fb0c: mov x0, x7 : curr->vruntime += calc_delta_fair(delta_exec, curr); 0.00 : ffff80001011fb10: add x1, x1, x2 0.05 : ffff80001011fb14: str x1, [x19, #88] : update_min_vruntime(cfs_rq); 0.00 : ffff80001011fb18: bl ffff80001011c438 : : if (entity_is_task(curr)) { 0.00 : ffff80001011fb1c: ldr x0, [x19, #136] 0.00 : ffff80001011fb20: cbz x0, ffff80001011fb34 0.00 : ffff80001011fb24: ldr x20, [x29, #24] : cgroup_account_cputime(curtask, delta_exec); : account_group_exec_runtime(curtask, delta_exec); : } : : account_cfs_rq_runtime(cfs_rq, delta_exec); : } 34.40 : ffff80001011fb28: ldr x19, [sp, #16] 49.16 : ffff80001011fb2c: ldp x29, x30, [sp], #48 0.00 : ffff80001011fb30: ret 0.02 : ffff80001011fb34: str x21, [x29, #32] : task_of(): : return container_of(se, struct task_struct, se); 0.00 : ffff80001011fb38: sub x21, x19, #0xc0 : cgroup_account_cputime(): : static inline void cgroup_account_cputime(struct task_struct *task, : u64 delta_exec) : { : struct cgroup *cgrp; : : cpuacct_charge(task, delta_exec); 0.00 : ffff80001011fb3c: mov x1, x20 0.02 : ffff80001011fb40: mov x0, x21 0.02 : ffff80001011fb44: bl ffff800010136480 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001011fb48: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011fb4c: ldr x0, [x19, #1808] : task_dfl_cgroup(): : return task_css_set(task)->dfl_cgrp; 0.02 : ffff80001011fb50: ldr x0, [x0, #88] : cgroup_parent(): : if (parent_css) 0.04 : ffff80001011fb54: ldr x1, [x0, #232] 0.00 : ffff80001011fb58: cbz x1, ffff80001011fb64 : cgroup_account_cputime(): : : rcu_read_lock(); : cgrp = task_dfl_cgroup(task); : if (cgroup_parent(cgrp)) : __cgroup_account_cputime(cgrp, delta_exec); 0.23 : ffff80001011fb5c: mov x1, x20 0.00 : ffff80001011fb60: bl ffff8000101973e8 <__cgroup_account_cputime> : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff80001011fb64: bl ffff800010160d20 <__rcu_read_unlock> : get_running_cputimer(): : */ : #ifdef CONFIG_POSIX_TIMERS : static inline : struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk) : { : struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 0.00 : ffff80001011fb68: ldr x0, [x21, #1640] : __read_once_size(): 0.00 : ffff80001011fb6c: ldr w1, [x0, #344] : get_running_cputimer(): : : /* : * Check whether posix CPU timers are active. If not the thread : * group accounting is not active either. Lockless check. : */ : if (!READ_ONCE(tsk->signal->posix_cputimers.timers_active)) 0.00 : ffff80001011fb70: cbz w1, ffff80001011fba0 : account_group_exec_runtime(): : static inline void account_group_exec_runtime(struct task_struct *tsk, : unsigned long long ns) : { : struct thread_group_cputimer *cputimer = get_running_cputimer(tsk); : : if (!cputimer) 0.00 : ffff80001011fb74: ldr x2, [x21, #1648] : get_running_cputimer(): : struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 0.00 : ffff80001011fb78: add x1, x0, #0xf8 : account_group_exec_runtime(): : if (!cputimer) 0.00 : ffff80001011fb7c: cmp x2, #0x0 0.00 : ffff80001011fb80: ccmp x1, #0x0, #0x4, ne // ne = any 0.00 : ffff80001011fb84: b.eq ffff80001011fba0 // b.none : return; : : atomic64_add(ns, &cputimer->cputime_atomic.sum_exec_runtime); 0.00 : ffff80001011fb88: add x1, x0, #0x108 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011fb8c: b ffff80001011fbc8 0.00 : ffff80001011fb90: b ffff80001011fbc8 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff80001011fb94: stadd x20, [x1] 0.00 : ffff80001011fb98: ldp x20, x21, [x29, #24] 0.00 : ffff80001011fb9c: b ffff80001011fb28 0.02 : ffff80001011fba0: ldp x20, x21, [x29, #24] : update_curr(): : } 0.00 : ffff80001011fba4: ldr x19, [sp, #16] 0.00 : ffff80001011fba8: ldp x29, x30, [sp], #48 0.00 : ffff80001011fbac: ret : calc_delta_fair(): : delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 0.00 : ffff80001011fbb0: mov x2, x19 0.00 : ffff80001011fbb4: mov x1, #0x100000 // #1048576 0.00 : ffff80001011fbb8: mov x0, x20 0.00 : ffff80001011fbbc: bl ffff80001011caa8 <__calc_delta> 0.00 : ffff80001011fbc0: mov x2, x0 0.00 : ffff80001011fbc4: b ffff80001011fb08 : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff80001011fbc8: add x0, x0, #0x108 0.00 : ffff80001011fbcc: b ffff8000101289ec 0.00 : ffff80001011fbd0: ldp x20, x21, [x29, #24] 0.00 : ffff80001011fbd4: b ffff80001011fb28 Percent | Source code & Disassembly of vmlinux for cycles (5510 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001014c410 <__handle_irq_event_percpu>: : __handle_irq_event_percpu(): : : wake_up_process(action->thread); : } : : irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc, unsigned int *flags) : { 2.32 : ffff80001014c410: stp x29, x30, [sp, #-112]! 0.00 : ffff80001014c414: mov x29, sp 1.39 : ffff80001014c418: str x25, [sp, #64] 0.02 : ffff80001014c41c: mov x25, x0 2.10 : ffff80001014c420: str x19, [sp, #16] 1.72 : ffff80001014c424: stp x21, x22, [sp, #32] 0.02 : ffff80001014c428: mov x22, x1 2.88 : ffff80001014c42c: stp x23, x24, [sp, #48] 0.00 : ffff80001014c430: adrp x24, ffff800011909000 0.05 : ffff80001014c434: add x0, x24, #0x908 : unsigned int irq = desc->irq_data.irq; : struct irqaction *action; : : record_irq_time(desc); : : for_each_action_of_desc(desc, action) { 1.47 : ffff80001014c438: ldr x19, [x25, #160] : { 3.97 : ffff80001014c43c: ldr x1, [x0] 1.80 : ffff80001014c440: str x1, [x29, #104] 0.00 : ffff80001014c444: mov x1, #0x0 // #0 : unsigned int irq = desc->irq_data.irq; 2.30 : ffff80001014c448: ldr w23, [x25, #92] : for_each_action_of_desc(desc, action) { 0.00 : ffff80001014c44c: cbz x19, ffff80001014c56c <__handle_irq_event_percpu+0x15c> 0.71 : ffff80001014c450: stp x26, x27, [x29, #72] : : trace_irq_handler_entry(irq, action); : res = action->handler(irq, action->dev_id); : trace_irq_handler_exit(irq, action, res); : : if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pS enabled interrupts\n", 0.02 : ffff80001014c454: adrp x26, ffff800011ad8000 0.27 : ffff80001014c458: adrp x27, ffff8000111c6000 0.00 : ffff80001014c45c: add x26, x26, #0x4bd 2.39 : ffff80001014c460: add x27, x27, #0x388 0.89 : ffff80001014c464: str x28, [x29, #88] : irqreturn_t retval = IRQ_NONE; 0.00 : ffff80001014c468: mov w21, #0x0 // #0 2.38 : ffff80001014c46c: str x20, [x29, #24] : arch_local_irq_disable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff80001014c470: mov x28, #0x60 // #96 0.00 : ffff80001014c474: nop : __handle_irq_event_percpu(): : res = action->handler(irq, action->dev_id); 0.42 : ffff80001014c478: ldp x2, x1, [x19] 0.00 : ffff80001014c47c: mov w0, w23 0.05 : ffff80001014c480: blr x2 0.05 : ffff80001014c484: mov w20, w0 : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.15 : ffff80001014c488: mrs x1, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.34 : ffff80001014c48c: and w0, w1, #0x80 : __handle_irq_event_percpu(): : if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pS enabled interrupts\n", 0.00 : ffff80001014c490: cbz w0, ffff80001014c510 <__handle_irq_event_percpu+0x100> : irq, action->handler)) : local_irq_disable(); : : switch (res) { 4.94 : ffff80001014c494: cmp w20, #0x1 0.00 : ffff80001014c498: b.eq ffff80001014c4b8 <__handle_irq_event_percpu+0xa8> // b.none 0.22 : ffff80001014c49c: cmp w20, #0x2 0.06 : ffff80001014c4a0: b.ne ffff80001014c4c8 <__handle_irq_event_percpu+0xb8> // b.any : case IRQ_WAKE_THREAD: : /* : * Catch drivers which return WAKE_THREAD but : * did not set up a thread function : */ : if (unlikely(!action->thread_fn)) { 20.28 : ffff80001014c4a4: ldr x0, [x19, #32] 0.07 : ffff80001014c4a8: cbz x0, ffff80001014c520 <__handle_irq_event_percpu+0x110> : warn_no_thread(irq, action); : break; : } : : __irq_wake_thread(desc, action); 5.71 : ffff80001014c4ac: mov x1, x19 0.00 : ffff80001014c4b0: mov x0, x25 0.00 : ffff80001014c4b4: bl ffff80001014c358 <__irq_wake_thread> : : /* Fall through - to add to randomness */ : case IRQ_HANDLED: : *flags |= action->flags; 2.99 : ffff80001014c4b8: ldr w1, [x19, #60] 7.75 : ffff80001014c4bc: ldr w0, [x22] 0.00 : ffff80001014c4c0: orr w0, w0, w1 0.20 : ffff80001014c4c4: str w0, [x22] : for_each_action_of_desc(desc, action) { 2.83 : ffff80001014c4c8: ldr x19, [x19, #24] : : default: : break; : } : : retval |= res; 0.93 : ffff80001014c4cc: orr w21, w21, w20 : for_each_action_of_desc(desc, action) { 0.00 : ffff80001014c4d0: cbnz x19, ffff80001014c478 <__handle_irq_event_percpu+0x68> 1.95 : ffff80001014c4d4: ldr x20, [x29, #24] 2.00 : ffff80001014c4d8: ldp x26, x27, [x29, #72] 12.25 : ffff80001014c4dc: ldr x28, [x29, #88] : } : : return retval; : } 0.00 : ffff80001014c4e0: add x24, x24, #0x908 0.00 : ffff80001014c4e4: mov w0, w21 2.00 : ffff80001014c4e8: ldr x2, [x29, #104] 0.60 : ffff80001014c4ec: ldr x1, [x24] 0.00 : ffff80001014c4f0: eor x1, x2, x1 0.00 : ffff80001014c4f4: cbnz x1, ffff80001014c594 <__handle_irq_event_percpu+0x184> 0.61 : ffff80001014c4f8: ldr x19, [sp, #16] 3.54 : ffff80001014c4fc: ldp x21, x22, [sp, #32] 2.32 : ffff80001014c500: ldp x23, x24, [sp, #48] 0.42 : ffff80001014c504: ldr x25, [sp, #64] 0.61 : ffff80001014c508: ldp x29, x30, [sp], #112 0.00 : ffff80001014c50c: ret : if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pS enabled interrupts\n", 0.00 : ffff80001014c510: ldrb w0, [x26] 0.00 : ffff80001014c514: cbz w0, ffff80001014c574 <__handle_irq_event_percpu+0x164> : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001014c518: msr daifset, #0x2 0.00 : ffff80001014c51c: b ffff80001014c494 <__handle_irq_event_percpu+0x84> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001014c520: ldr x0, [x19, #64] : test_and_set_bit(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (READ_ONCE(*p) & mask) 0.00 : ffff80001014c524: tbnz w0, #1, ffff80001014c4c8 <__handle_irq_event_percpu+0xb8> : warn_no_thread(): : if (test_and_set_bit(IRQTF_WARNED, &action->thread_flags)) 0.00 : ffff80001014c528: add x0, x19, #0x40 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001014c52c: b ffff80001014c55c <__handle_irq_event_percpu+0x14c> 0.00 : ffff80001014c530: b ffff80001014c55c <__handle_irq_event_percpu+0x14c> : __lse_atomic64_fetch_or(): : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff80001014c534: mov x1, #0x2 // #2 0.00 : ffff80001014c538: ldsetal x1, x1, [x0] 0.00 : ffff80001014c53c: mov x0, x1 : warn_no_thread(): 0.00 : ffff80001014c540: tbnz w0, #1, ffff80001014c4c8 <__handle_irq_event_percpu+0xb8> : printk(KERN_WARNING "IRQ %d device %s returned IRQ_WAKE_THREAD " 0.00 : ffff80001014c544: ldr x2, [x19, #80] 0.00 : ffff80001014c548: adrp x0, ffff8000111c6000 0.00 : ffff80001014c54c: mov w1, w23 0.00 : ffff80001014c550: add x0, x0, #0x3b0 0.00 : ffff80001014c554: bl ffff80001014a85c 0.00 : ffff80001014c558: b ffff80001014c4c8 <__handle_irq_event_percpu+0xb8> : __ll_sc_atomic64_fetch_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff80001014c55c: b ffff80001014c6d8 0.00 : ffff80001014c560: mov x0, x1 : warn_no_thread(): : if (test_and_set_bit(IRQTF_WARNED, &action->thread_flags)) 0.00 : ffff80001014c564: tbnz w0, #1, ffff80001014c4c8 <__handle_irq_event_percpu+0xb8> 0.00 : ffff80001014c568: b ffff80001014c544 <__handle_irq_event_percpu+0x134> : __handle_irq_event_percpu(): : irqreturn_t retval = IRQ_NONE; 0.00 : ffff80001014c56c: mov w21, #0x0 // #0 : return retval; 0.00 : ffff80001014c570: b ffff80001014c4e0 <__handle_irq_event_percpu+0xd0> : if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pS enabled interrupts\n", 0.00 : ffff80001014c574: ldr x2, [x19] 0.00 : ffff80001014c578: mov w0, #0x1 // #1 0.00 : ffff80001014c57c: mov w1, w23 0.00 : ffff80001014c580: strb w0, [x26] 0.00 : ffff80001014c584: mov x0, x27 0.00 : ffff80001014c588: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff80001014c58c: brk #0x800 0.00 : ffff80001014c590: b ffff80001014c518 <__handle_irq_event_percpu+0x108> 0.00 : ffff80001014c594: str x20, [x29, #24] 0.00 : ffff80001014c598: stp x26, x27, [x29, #72] 0.00 : ffff80001014c59c: str x28, [x29, #88] : } 0.00 : ffff80001014c5a0: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (5200 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100eccc0 : : irq_exit(): : : /* : * Exit an interrupt context. Process softirqs if needed and possible: : */ : void irq_exit(void) : { 0.16 : ffff8000100eccc0: stp x29, x30, [sp, #-32]! 0.00 : ffff8000100eccc4: mov x29, sp 5.81 : ffff8000100eccc8: str x19, [sp, #16] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.32 : ffff8000100ecccc: mrs x19, sp_el0 : account_irq_exit_time(): : } : : static inline void account_irq_exit_time(struct task_struct *tsk) : { : vtime_account_irq_exit(tsk); : irqtime_account_irq(tsk); 0.00 : ffff8000100eccd0: mov x0, x19 0.00 : ffff8000100eccd4: bl ffff80001011b180 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.67 : ffff8000100eccd8: ldr w0, [x19, #16] : __preempt_count_sub(): : } : : static inline void __preempt_count_sub(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc -= val; 0.00 : ffff8000100eccdc: sub w0, w0, #0x10, lsl #12 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.22 : ffff8000100ecce0: str w0, [x19, #16] : __read_once_size(): : __READ_ONCE_SIZE; 31.95 : ffff8000100ecce4: ldr w0, [x19, #16] : irq_exit(): : #else : lockdep_assert_irqs_disabled(); : #endif : account_irq_exit_time(current); : preempt_count_sub(HARDIRQ_OFFSET); : if (!in_interrupt() && local_softirq_pending()) 0.00 : ffff8000100ecce8: tst w0, #0x1fff00 0.00 : ffff8000100eccec: b.ne ffff8000100ecd70 // b.any 29.98 : ffff8000100eccf0: adrp x0, ffff800011528000 : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff8000100eccf4: mov x1, sp : irq_exit(): 0.00 : ffff8000100eccf8: add x0, x0, #0x7c0 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 4.44 : ffff8000100eccfc: mrs x2, tpidr_el1 : irq_exit(): 0.02 : ffff8000100ecd00: ldr w3, [x0, x2] 0.00 : ffff8000100ecd04: cbnz w3, ffff8000100ecd4c : tick_irq_exit(): : int cpu = smp_processor_id(); 9.46 : ffff8000100ecd08: adrp x0, ffff80001151d000 0.00 : ffff8000100ecd0c: add x0, x0, #0x18 : __my_cpu_offset(): 3.66 : ffff8000100ecd10: mrs x1, tpidr_el1 : tick_irq_exit(): : if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) { 2.28 : ffff8000100ecd14: ldr w0, [x0, x1] 0.00 : ffff8000100ecd18: bl ffff800010118820 4.23 : ffff8000100ecd1c: cbz w0, ffff8000100ecd3c : get_current(): 0.00 : ffff8000100ecd20: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000100ecd24: ldr x1, [x0] : tick_irq_exit(): 0.00 : ffff8000100ecd28: tst w1, #0x2 0.00 : ffff8000100ecd2c: b.ne ffff8000100ecd3c // b.any : __read_once_size(): 0.00 : ffff8000100ecd30: ldr w0, [x0, #16] : tick_irq_exit(): : if (!in_irq()) 0.00 : ffff8000100ecd34: tst w0, #0xf0000 0.00 : ffff8000100ecd38: b.eq ffff8000100ecd78 // b.none : irq_exit(): : invoke_softirq(); : : tick_irq_exit(); : rcu_irq_exit(); 0.16 : ffff8000100ecd3c: bl ffff800010160fc0 : trace_hardirq_exit(); /* must be last! */ : } 5.30 : ffff8000100ecd40: ldr x19, [sp, #16] 0.25 : ffff8000100ecd44: ldp x29, x30, [sp], #32 0.00 : ffff8000100ecd48: ret : invoke_softirq(): : if (ksoftirqd_running(local_softirq_pending())) 0.02 : ffff8000100ecd4c: mov x0, x3 0.00 : ffff8000100ecd50: bl ffff8000100ec5f8 0.02 : ffff8000100ecd54: tst w0, #0xff 0.00 : ffff8000100ecd58: b.ne ffff8000100ecd70 // b.any : if (!force_irqthreads) { 0.00 : ffff8000100ecd5c: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000100ecd60: ldrb w0, [x0, #640] 0.00 : ffff8000100ecd64: cbz w0, ffff8000100ecd8c : wakeup_softirqd(); 0.00 : ffff8000100ecd68: bl ffff8000100ec5c0 0.00 : ffff8000100ecd6c: nop 0.06 : ffff8000100ecd70: mov x1, sp 0.00 : ffff8000100ecd74: b ffff8000100ecd08 : tick_irq_exit(): : tick_nohz_irq_exit(); 0.00 : ffff8000100ecd78: bl ffff80001017be78 : irq_exit(): : rcu_irq_exit(); 0.00 : ffff8000100ecd7c: bl ffff800010160fc0 : } 0.00 : ffff8000100ecd80: ldr x19, [sp, #16] 0.00 : ffff8000100ecd84: ldp x29, x30, [sp], #32 0.00 : ffff8000100ecd88: ret : do_softirq_own_stack(): : #ifdef __ARCH_HAS_DO_SOFTIRQ : void do_softirq_own_stack(void); : #else : static inline void do_softirq_own_stack(void) : { : __do_softirq(); 0.00 : ffff8000100ecd8c: bl ffff800010081000 <__do_softirq> 0.00 : ffff8000100ecd90: mov x1, sp 0.00 : ffff8000100ecd94: b ffff8000100ecd08 Percent | Source code & Disassembly of vmlinux for cycles (4892 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010115000 : : activate_task(): : uclamp_rq_dec(rq, p); : p->sched_class->dequeue_task(rq, p, flags); : } : : void activate_task(struct rq *rq, struct task_struct *p, int flags) : { 0.00 : ffff800010115000: stp x29, x30, [sp, #-48]! 0.00 : ffff800010115004: mov x29, sp 0.02 : ffff800010115008: stp x19, x20, [sp, #16] 0.00 : ffff80001011500c: mov x20, x0 0.00 : ffff800010115010: mov x19, x1 : if (task_contributes_to_load(p)) 0.00 : ffff800010115014: ldr x0, [x1, #24] 0.00 : ffff800010115018: tbz w0, #1, ffff800010115038 0.00 : ffff80001011501c: ldr w0, [x1, #44] 0.00 : ffff800010115020: tbnz w0, #16, ffff800010115038 0.00 : ffff800010115024: ldr x0, [x1, #24] 0.00 : ffff800010115028: tbnz w0, #10, ffff800010115038 : rq->nr_uninterruptible--; 0.00 : ffff80001011502c: ldr x0, [x20, #2344] 0.00 : ffff800010115030: sub x0, x0, #0x1 0.00 : ffff800010115034: str x0, [x20, #2344] : enqueue_task(): : if (!(flags & ENQUEUE_NOCLOCK)) 4.80 : ffff800010115038: tbnz w2, #3, ffff800010115044 : update_rq_clock(): : if (rq->clock_update_flags & RQCF_ACT_SKIP) 0.00 : ffff80001011503c: ldr w0, [x20, #2392] 0.00 : ffff800010115040: tbz w0, #1, ffff800010115090 : enqueue_task(): : if (!(flags & ENQUEUE_RESTORE)) { 0.00 : ffff800010115044: tbnz w2, #1, ffff800010115054 : sched_info_on(): : { : #ifdef CONFIG_SCHEDSTATS : return 1; : #elif defined(CONFIG_TASK_DELAY_ACCT) : extern int delayacct_on; : return delayacct_on; 0.00 : ffff800010115048: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> : sched_info_queued(): : * the timestamp if it is already not set. It's assumed that : * sched_info_dequeued() will clear that stamp when appropriate. : */ : static inline void sched_info_queued(struct rq *rq, struct task_struct *t) : { : if (sched_info_on()) { 0.04 : ffff80001011504c: ldr w0, [x0, #792] 0.00 : ffff800010115050: cbnz w0, ffff80001011507c : enqueue_task(): : p->sched_class->enqueue_task(rq, p, flags); 0.00 : ffff800010115054: ldr x3, [x19, #128] 0.00 : ffff800010115058: mov x1, x19 0.00 : ffff80001011505c: mov x0, x20 2.97 : ffff800010115060: ldr x3, [x3, #8] 0.00 : ffff800010115064: blr x3 : activate_task(): : : enqueue_task(rq, p, flags); : : p->on_rq = TASK_ON_RQ_QUEUED; 0.80 : ffff800010115068: mov w0, #0x1 // #1 3.59 : ffff80001011506c: str w0, [x19, #104] : } 0.27 : ffff800010115070: ldp x19, x20, [sp, #16] 0.59 : ffff800010115074: ldp x29, x30, [sp], #48 0.10 : ffff800010115078: ret : sched_info_queued(): : if (!t->sched_info.last_queued) 5.17 : ffff80001011507c: ldr x0, [x19, #864] 0.00 : ffff800010115080: cbnz x0, ffff800010115054 : t->sched_info.last_queued = rq_clock(rq); 80.06 : ffff800010115084: ldr x0, [x20, #2400] 1.59 : ffff800010115088: str x0, [x19, #864] 0.00 : ffff80001011508c: b ffff800010115054 0.00 : ffff800010115090: str w2, [x29, #44] : update_rq_clock(): 0.00 : ffff800010115094: mov x0, x20 0.00 : ffff800010115098: bl ffff800010113d48 0.00 : ffff80001011509c: ldr w2, [x29, #44] 0.00 : ffff8000101150a0: b ffff800010115044 Percent | Source code & Disassembly of vmlinux for cycles (4965 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010113d48 : : update_rq_clock(): : update_irq_load_avg(rq, irq_delta + steal); : #endif : update_rq_clock_pelt(rq, delta); : } : : void update_rq_clock(struct rq *rq) 0.65 : ffff800010113d48: stp x29, x30, [sp, #-48]! 0.00 : ffff800010113d4c: mov x29, sp 4.72 : ffff800010113d50: str x20, [sp, #24] 0.00 : ffff800010113d54: mov x20, x0 : if (sched_feat(WARN_DOUBLE_CLOCK)) : SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); : rq->clock_update_flags |= RQCF_UPDATED; : #endif : : delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 0.26 : ffff800010113d58: ldr w0, [x0, #2568] 0.06 : ffff800010113d5c: bl ffff80001011b140 0.00 : ffff800010113d60: ldr x1, [x20, #2400] : if (delta < 0) 0.00 : ffff800010113d64: subs x2, x0, x1 0.00 : ffff800010113d68: b.mi ffff800010113e10 // b.first 5.35 : ffff800010113d6c: str x19, [x29, #16] 0.00 : ffff800010113d70: str x21, [x29, #32] : irq_time_read(): : * Otherwise ksoftirqd's sum_exec_runtime is substracted its own runtime : * and never move forward. : */ : static inline u64 irq_time_read(int cpu) : { : struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu); 0.00 : ffff800010113d74: adrp x21, ffff800011909000 0.00 : ffff800010113d78: add x1, x21, #0x928 : update_rq_clock(): : return; : rq->clock += delta; 0.00 : ffff800010113d7c: str x0, [x20, #2400] : irq_time_read(): 0.00 : ffff800010113d80: ldrsw x4, [x20, #2568] 0.00 : ffff800010113d84: adrp x0, ffff80001151f000 : update_rq_clock_task(): : irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 4.76 : ffff800010113d88: ldr x3, [x20, #2840] : irq_time_read(): 0.00 : ffff800010113d8c: add x0, x0, #0x40 : unsigned int seq; : u64 total; : : do { : seq = __u64_stats_fetch_begin(&irqtime->sync); : total = irqtime->total; 4.99 : ffff800010113d90: ldr x1, [x1, x4, lsl #3] : update_rq_clock_task(): : rq->clock_task += delta; 0.78 : ffff800010113d94: ldr x4, [x20, #2432] : irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 0.02 : ffff800010113d98: ldr x1, [x1, x0] 0.00 : ffff800010113d9c: sub x1, x1, x3 0.00 : ffff800010113da0: cmp x2, x1 0.00 : ffff800010113da4: csel x1, x2, x1, le : delta -= irq_delta; 8.17 : ffff800010113da8: sub x19, x2, x1 : rq->prev_irq_time += irq_delta; 0.00 : ffff800010113dac: add x3, x3, x1 : rq->clock_task += delta; 0.00 : ffff800010113db0: add x0, x4, x19 0.10 : ffff800010113db4: str x0, [x20, #2432] : rq->prev_irq_time += irq_delta; 0.00 : ffff800010113db8: str x3, [x20, #2840] : if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 0.00 : ffff800010113dbc: cbnz x1, ffff800010113e1c : is_idle_task(): : * : * Return: 1 if @p is an idle task. 0 otherwise. : */ : static inline bool is_idle_task(const struct task_struct *p) : { : return !!(p->flags & PF_IDLE); 0.02 : ffff800010113dc0: ldr x0, [x20, #2352] 13.91 : ffff800010113dc4: ldr w0, [x0, #44] : update_rq_clock_pelt(): : * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 : * : */ : static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) : { : if (unlikely(is_idle_task(rq->curr))) { 0.00 : ffff800010113dc8: tbnz w0, #1, ffff800010113e30 : topology_get_cpu_scale(): : : struct sched_domain; : static inline : unsigned long topology_get_cpu_scale(int cpu) : { : return per_cpu(cpu_scale, cpu); 1.65 : ffff800010113dcc: ldrsw x0, [x20, #2568] 0.00 : ffff800010113dd0: add x21, x21, #0x928 0.00 : ffff800010113dd4: adrp x2, ffff800011527000 0.00 : ffff800010113dd8: add x2, x2, #0xa00 : topology_get_freq_scale(): : DECLARE_PER_CPU(unsigned long, freq_scale); : : static inline : unsigned long topology_get_freq_scale(int cpu) : { : return per_cpu(freq_scale, cpu); 0.19 : ffff800010113ddc: adrp x1, ffff800011527000 0.00 : ffff800010113de0: add x1, x1, #0xa08 : topology_get_cpu_scale(): : return per_cpu(cpu_scale, cpu); 0.02 : ffff800010113de4: ldr x3, [x21, x0, lsl #3] : update_rq_clock_pelt(): : * computation : */ : delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); : delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); : : rq->clock_pelt += delta; 0.12 : ffff800010113de8: ldr x4, [x20, #2440] : delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); 0.02 : ffff800010113dec: ldr x0, [x2, x3] : delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); 0.69 : ffff800010113df0: ldr x1, [x3, x1] : delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); 0.00 : ffff800010113df4: mul x0, x19, x0 0.00 : ffff800010113df8: lsr x0, x0, #10 : delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); 0.00 : ffff800010113dfc: mul x0, x0, x1 : rq->clock_pelt += delta; 0.04 : ffff800010113e00: add x0, x4, x0, lsr #10 0.02 : ffff800010113e04: str x0, [x20, #2440] 0.00 : ffff800010113e08: ldr x19, [x29, #16] 0.25 : ffff800010113e0c: ldr x21, [x29, #32] : update_rq_clock(): : update_rq_clock_task(rq, delta); : } 0.06 : ffff800010113e10: ldr x20, [sp, #24] 4.80 : ffff800010113e14: ldp x29, x30, [sp], #48 0.00 : ffff800010113e18: ret : update_rq_clock_task(): : update_irq_load_avg(rq, irq_delta + steal); 0.64 : ffff800010113e1c: mov x0, x20 0.04 : ffff800010113e20: bl ffff800010134fe0 : is_idle_task(): 0.15 : ffff800010113e24: ldr x0, [x20, #2352] 0.11 : ffff800010113e28: ldr w0, [x0, #44] : update_rq_clock_pelt(): : if (unlikely(is_idle_task(rq->curr))) { 0.00 : ffff800010113e2c: tbz w0, #1, ffff800010113dcc : rq->clock_pelt = rq_clock_task(rq); 22.60 : ffff800010113e30: ldr x0, [x20, #2432] 24.80 : ffff800010113e34: str x0, [x20, #2440] 0.00 : ffff800010113e38: ldr x19, [x29, #16] 0.02 : ffff800010113e3c: ldr x21, [x29, #32] 0.00 : ffff800010113e40: b ffff800010113e10 Percent | Source code & Disassembly of vmlinux for cycles (4768 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107d35e0 : : nvme_unmap_data(): : return false; : return true; : } : : static void nvme_unmap_data(struct nvme_dev *dev, struct request *req) : { 0.40 : ffff8000107d35e0: stp x29, x30, [sp, #-64]! 0.00 : ffff8000107d35e4: mov x29, sp 2.22 : ffff8000107d35e8: str x19, [sp, #16] : struct nvme_iod *iod = blk_mq_rq_to_pdu(req); : const int last_prp = dev->ctrl.page_size / sizeof(__le64) - 1; : dma_addr_t dma_addr = iod->first_dma, next_dma_addr; 0.25 : ffff8000107d35ec: add x19, x1, #0x118 : { 0.50 : ffff8000107d35f0: stp x21, x22, [sp, #32] 0.57 : ffff8000107d35f4: mov x21, x1 3.04 : ffff8000107d35f8: str x24, [sp, #56] 0.31 : ffff8000107d35fc: mov x22, x0 : int i; : : if (iod->dma_len) { 0.51 : ffff8000107d3600: ldr w2, [x19, #64] : dma_addr_t dma_addr = iod->first_dma, next_dma_addr; 31.93 : ffff8000107d3604: ldr x24, [x19, #56] : if (iod->dma_len) { 0.06 : ffff8000107d3608: cbz w2, ffff8000107d3654 : dma_unmap_page(dev->dev, dma_addr, iod->dma_len, 0.87 : ffff8000107d360c: ldr x0, [x0, #336] 4.77 : ffff8000107d3610: mov w2, w2 : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 7.61 : ffff8000107d3614: ldr w3, [x1, #24] : dma_unmap_page_attrs(): : } : : static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, : size_t size, enum dma_data_direction dir, unsigned long attrs) : { : const struct dma_map_ops *ops = get_dma_ops(dev); 0.27 : ffff8000107d3618: ldr x1, [x0, #576] : nvme_unmap_data(): 3.36 : ffff8000107d361c: tst x3, #0x1 4.04 : ffff8000107d3620: cset w3, eq // eq = none 0.44 : ffff8000107d3624: add w3, w3, #0x1 : get_dma_ops(): : if (dev->dma_ops) 12.38 : ffff8000107d3628: cbz x1, ffff8000107d36a0 : dma_unmap_page_attrs(): : : BUG_ON(!valid_dma_direction(dir)); : if (dma_is_direct(ops)) : dma_direct_unmap_page(dev, addr, size, dir, attrs); : else if (ops->unmap_page) 2.11 : ffff8000107d362c: ldr x5, [x1, #40] 9.55 : ffff8000107d3630: cbz x5, ffff8000107d3640 : ops->unmap_page(dev, addr, size, dir, attrs); 10.64 : ffff8000107d3634: mov x4, #0x0 // #0 0.25 : ffff8000107d3638: mov x1, x24 0.58 : ffff8000107d363c: blr x5 : nvme_unmap_data(): : dma_pool_free(dev->prp_page_pool, addr, dma_addr); : dma_addr = next_dma_addr; : } : : mempool_free(iod->sg, dev->iod_mempool); : } 0.25 : ffff8000107d3640: ldr x19, [sp, #16] 1.61 : ffff8000107d3644: ldp x21, x22, [sp, #32] 1.29 : ffff8000107d3648: ldr x24, [sp, #56] 0.18 : ffff8000107d364c: ldp x29, x30, [sp], #64 0.00 : ffff8000107d3650: ret 0.00 : ffff8000107d3654: str x23, [x29, #48] : WARN_ON_ONCE(!iod->nents); 0.00 : ffff8000107d3658: ldr w2, [x19, #52] : const int last_prp = dev->ctrl.page_size / sizeof(__le64) - 1; 0.00 : ffff8000107d365c: ldr w23, [x0, #1720] : WARN_ON_ONCE(!iod->nents); 0.00 : ffff8000107d3660: cbz w2, ffff8000107d37a4 : dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req)); 0.00 : ffff8000107d3664: ldr x0, [x22, #336] : op_is_write(): 0.00 : ffff8000107d3668: ldr w3, [x21, #24] : nvme_unmap_data(): : if (is_pci_p2pdma_page(sg_page(iod->sg))) 0.00 : ffff8000107d366c: ldr x4, [x19, #80] : dma_unmap_sg_attrs(): : : static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, : int nents, enum dma_data_direction dir, : unsigned long attrs) : { : const struct dma_map_ops *ops = get_dma_ops(dev); 0.00 : ffff8000107d3670: ldr x1, [x0, #576] : nvme_unmap_data(): : dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req)); 0.00 : ffff8000107d3674: tst x3, #0x1 0.00 : ffff8000107d3678: cset w3, eq // eq = none 0.00 : ffff8000107d367c: add w3, w3, #0x1 : get_dma_ops(): : if (dev->dma_ops) 0.00 : ffff8000107d3680: cbz x1, ffff8000107d36c0 : dma_unmap_sg_attrs(): : : BUG_ON(!valid_dma_direction(dir)); : debug_dma_unmap_sg(dev, sg, nents, dir); : if (dma_is_direct(ops)) : dma_direct_unmap_sg(dev, sg, nents, dir, attrs); : else if (ops->unmap_sg) 0.00 : ffff8000107d3684: ldr x5, [x1, #56] 0.00 : ffff8000107d3688: cbz x5, ffff8000107d36d0 : ops->unmap_sg(dev, sg, nents, dir, attrs); 0.00 : ffff8000107d368c: mov x1, x4 0.00 : ffff8000107d3690: mov x4, #0x0 // #0 0.00 : ffff8000107d3694: blr x5 0.00 : ffff8000107d3698: ldr x4, [x19, #80] 0.00 : ffff8000107d369c: b ffff8000107d36d0 : dma_unmap_page_attrs(): : dma_direct_unmap_page(dev, addr, size, dir, attrs); 0.00 : ffff8000107d36a0: mov x1, x24 0.00 : ffff8000107d36a4: mov x4, #0x0 // #0 0.00 : ffff8000107d36a8: bl ffff800010163b00 : nvme_unmap_data(): : } 0.00 : ffff8000107d36ac: ldr x19, [sp, #16] 0.00 : ffff8000107d36b0: ldp x21, x22, [sp, #32] 0.00 : ffff8000107d36b4: ldr x24, [sp, #56] 0.00 : ffff8000107d36b8: ldp x29, x30, [sp], #64 0.00 : ffff8000107d36bc: ret : dma_unmap_sg_attrs(): : dma_direct_unmap_sg(dev, sg, nents, dir, attrs); 0.00 : ffff8000107d36c0: mov x1, x4 0.00 : ffff8000107d36c4: mov x4, #0x0 // #0 0.00 : ffff8000107d36c8: bl ffff800010163b88 0.00 : ffff8000107d36cc: ldr x4, [x19, #80] : nvme_unmap_data(): : if (iod->npages == 0) 0.00 : ffff8000107d36d0: ldr w0, [x19, #48] 0.00 : ffff8000107d36d4: cbnz w0, ffff8000107d36fc : blk_rq_nr_phys_segments(): : * own special payload. In that case we still return 1 here so that this : * special payload will be mapped. : */ : static inline unsigned short blk_rq_nr_phys_segments(struct request *rq) : { : if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 0.00 : ffff8000107d36d8: ldr w2, [x21, #28] 0.00 : ffff8000107d36dc: mov x1, #0x20 // #32 : nvme_unmap_data(): : dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0], 0.00 : ffff8000107d36e0: ldr x0, [x22, #352] : blk_rq_nr_phys_segments(): 0.00 : ffff8000107d36e4: tbz w2, #18, ffff8000107d3798 : nvme_unmap_data(): 0.00 : ffff8000107d36e8: ldr x1, [x4, x1] 0.00 : ffff8000107d36ec: mov x2, x24 0.00 : ffff8000107d36f0: bl ffff800010239ab0 0.00 : ffff8000107d36f4: ldr w0, [x19, #48] 0.00 : ffff8000107d36f8: ldr x4, [x19, #80] : for (i = 0; i < iod->npages; i++) { 0.00 : ffff8000107d36fc: cmp w0, #0x0 0.00 : ffff8000107d3700: b.le ffff8000107d3774 : const int last_prp = dev->ctrl.page_size / sizeof(__le64) - 1; 0.00 : ffff8000107d3704: lsr w23, w23, #3 0.00 : ffff8000107d3708: str x20, [x29, #24] 0.00 : ffff8000107d370c: sub w23, w23, #0x1 : next_dma_addr = le64_to_cpu(prp_list[last_prp]); 0.00 : ffff8000107d3710: mov w20, #0x0 // #0 0.00 : ffff8000107d3714: sbfiz x23, x23, #3, #32 0.00 : ffff8000107d3718: b ffff8000107d3744 : next_dma_addr = 0.00 : ffff8000107d371c: ldr x3, [x1, #4080] : dma_pool_free(dev->prp_page_pool, addr, dma_addr); 0.00 : ffff8000107d3720: ldr x0, [x22, #344] 0.00 : ffff8000107d3724: mov x2, x24 : dma_addr = next_dma_addr; 0.00 : ffff8000107d3728: mov x24, x3 : for (i = 0; i < iod->npages; i++) { 0.00 : ffff8000107d372c: add w20, w20, #0x1 : dma_pool_free(dev->prp_page_pool, addr, dma_addr); 0.00 : ffff8000107d3730: bl ffff800010239ab0 : for (i = 0; i < iod->npages; i++) { 0.00 : ffff8000107d3734: ldr w0, [x19, #48] 0.00 : ffff8000107d3738: ldr x4, [x19, #80] 0.00 : ffff8000107d373c: cmp w0, w20 0.00 : ffff8000107d3740: b.le ffff8000107d3770 : blk_rq_nr_phys_segments(): 0.00 : ffff8000107d3744: ldr w0, [x21, #28] 0.00 : ffff8000107d3748: mov x1, #0x20 // #32 0.00 : ffff8000107d374c: tbnz w0, #18, ffff8000107d3758 0.00 : ffff8000107d3750: ldrh w1, [x21, #194] 0.00 : ffff8000107d3754: lsl x1, x1, #5 : nvme_unmap_data(): : if (iod->use_sgl) { 0.00 : ffff8000107d3758: ldrb w0, [x19, #40] : void *addr = nvme_pci_iod_list(req)[i]; 0.00 : ffff8000107d375c: add x1, x1, w20, sxtw #3 0.00 : ffff8000107d3760: ldr x1, [x4, x1] : if (iod->use_sgl) { 0.00 : ffff8000107d3764: cbnz w0, ffff8000107d371c : next_dma_addr = le64_to_cpu(prp_list[last_prp]); 0.00 : ffff8000107d3768: ldr x3, [x1, x23] 0.00 : ffff8000107d376c: b ffff8000107d3720 0.00 : ffff8000107d3770: ldr x20, [x29, #24] : mempool_free(iod->sg, dev->iod_mempool); 0.00 : ffff8000107d3774: ldr x1, [x22, #3176] 0.00 : ffff8000107d3778: mov x0, x4 0.00 : ffff8000107d377c: bl ffff8000101d7548 0.00 : ffff8000107d3780: ldr x23, [x29, #48] : } 0.00 : ffff8000107d3784: ldr x19, [sp, #16] 0.00 : ffff8000107d3788: ldp x21, x22, [sp, #32] 0.00 : ffff8000107d378c: ldr x24, [sp, #56] 0.00 : ffff8000107d3790: ldp x29, x30, [sp], #64 0.00 : ffff8000107d3794: ret 0.00 : ffff8000107d3798: ldrh w1, [x21, #194] 0.00 : ffff8000107d379c: lsl x1, x1, #5 0.00 : ffff8000107d37a0: b ffff8000107d36e8 : WARN_ON_ONCE(!iod->nents); 0.00 : ffff8000107d37a4: brk #0x800 0.00 : ffff8000107d37a8: ldr w2, [x19, #52] 0.00 : ffff8000107d37ac: b ffff8000107d3664 Percent | Source code & Disassembly of vmlinux for cycles (4754 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001014c5a8 : : handle_irq_event_percpu(): : : return retval; : } : : irqreturn_t handle_irq_event_percpu(struct irq_desc *desc) : { 2.84 : ffff80001014c5a8: stp x29, x30, [sp, #-64]! 0.00 : ffff80001014c5ac: mov x29, sp 2.97 : ffff80001014c5b0: stp x19, x20, [sp, #16] : irqreturn_t retval; : unsigned int flags = 0; 0.59 : ffff80001014c5b4: add x1, x29, #0x40 : { 4.28 : ffff80001014c5b8: str x21, [sp, #32] 0.09 : ffff80001014c5bc: adrp x19, ffff800011909000 0.00 : ffff80001014c5c0: add x2, x19, #0x908 0.00 : ffff80001014c5c4: mov x21, x0 1.49 : ffff80001014c5c8: ldr x3, [x2] 9.99 : ffff80001014c5cc: str x3, [x29, #56] 0.00 : ffff80001014c5d0: mov x3, #0x0 // #0 : unsigned int flags = 0; 1.75 : ffff80001014c5d4: str wzr, [x1, #-12]! : : retval = __handle_irq_event_percpu(desc, &flags); 0.04 : ffff80001014c5d8: bl ffff80001014c410 <__handle_irq_event_percpu> 4.19 : ffff80001014c5dc: mov w20, w0 : : add_interrupt_randomness(desc->irq_data.irq, flags); 6.15 : ffff80001014c5e0: ldr w1, [x29, #52] 11.58 : ffff80001014c5e4: ldr w0, [x21, #92] 0.00 : ffff80001014c5e8: bl ffff800010703840 : : if (!noirqdebug) 0.16 : ffff80001014c5ec: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 3.61 : ffff80001014c5f0: ldrb w0, [x0, #644] 0.00 : ffff80001014c5f4: cbnz w0, ffff80001014c604 : note_interrupt(desc, retval); 2.21 : ffff80001014c5f8: mov w1, w20 0.00 : ffff80001014c5fc: mov x0, x21 0.00 : ffff80001014c600: bl ffff8000101505c8 : return retval; : } 1.02 : ffff80001014c604: add x19, x19, #0x908 0.00 : ffff80001014c608: mov w0, w20 5.14 : ffff80001014c60c: ldr x2, [x29, #56] 17.91 : ffff80001014c610: ldr x1, [x19] 0.00 : ffff80001014c614: eor x1, x2, x1 0.00 : ffff80001014c618: cbnz x1, ffff80001014c62c 12.22 : ffff80001014c61c: ldp x19, x20, [sp, #16] 11.73 : ffff80001014c620: ldr x21, [sp, #32] 0.06 : ffff80001014c624: ldp x29, x30, [sp], #64 0.00 : ffff80001014c628: ret 0.00 : ffff80001014c62c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (4478 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cb92d8 : : cpumask_next(): : * @srcp: the cpumask pointer : * : * Returns >= nr_cpu_ids if no further cpus set. : */ : unsigned int cpumask_next(int n, const struct cpumask *srcp) : { 1.12 : ffff800010cb92d8: stp x29, x30, [sp, #-16]! : /* -1 is a legal arg here. */ : if (n != -1) : cpumask_check(n); : return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); 0.00 : ffff800010cb92dc: add w2, w0, #0x1 : { 0.09 : ffff800010cb92e0: mov x0, x1 : return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); 0.16 : ffff800010cb92e4: mov x1, #0x100 // #256 : { 95.47 : ffff800010cb92e8: mov x29, sp : return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); 0.00 : ffff800010cb92ec: sxtw x2, w2 0.20 : ffff800010cb92f0: bl ffff80001048daa8 : } 2.62 : ffff800010cb92f4: ldp x29, x30, [sp], #16 0.33 : ffff800010cb92f8: ret Percent | Source code & Disassembly of vmlinux for cycles (7785 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010130ab0 : : cpupri_set(): : * : * Returns: (void) : */ : void cpupri_set(struct cpupri *cp, int cpu, int newpri) : { : int *currpri = &cp->cpu_to_pri[cpu]; 0.18 : ffff800010130ab0: sxtw x8, w1 0.01 : ffff800010130ab4: ldr x9, [x0, #4080] : convert_prio(): : if (prio == CPUPRI_INVALID) 0.00 : ffff800010130ab8: cmn w2, #0x1 : cpupri_set(): : int oldpri = *currpri; 1.73 : ffff800010130abc: ldr w3, [x9, x8, lsl #2] : convert_prio(): : if (prio == CPUPRI_INVALID) 1.76 : ffff800010130ac0: b.eq ffff800010130ba0 // b.none : else if (prio == MAX_PRIO) 10.75 : ffff800010130ac4: cmp w2, #0x8c 0.00 : ffff800010130ac8: b.eq ffff800010130bf0 // b.none : else if (prio >= MAX_RT_PRIO) 0.00 : ffff800010130acc: cmp w2, #0x63 0.00 : ffff800010130ad0: b.le ffff800010130aec : cpupri_set(): : : newpri = convert_prio(newpri); : : BUG_ON(newpri >= CPUPRI_NR_PRIORITIES); : : if (newpri == oldpri) 0.00 : ffff800010130ad4: cmp w3, #0x1 0.00 : ffff800010130ad8: b.eq ffff800010130b9c // b.none 0.01 : ffff800010130adc: add x5, x0, #0x28 0.00 : ffff800010130ae0: mov x4, #0x30 // #48 0.00 : ffff800010130ae4: mov w2, #0x1 // #1 0.00 : ffff800010130ae8: b ffff800010130b14 : convert_prio(): : cpupri = MAX_RT_PRIO - prio + 1; 0.00 : ffff800010130aec: mov w4, #0x65 // #101 0.00 : ffff800010130af0: sub w2, w4, w2 : cpupri_set(): : BUG_ON(newpri >= CPUPRI_NR_PRIORITIES); 0.00 : ffff800010130af4: cmp w2, w4 0.00 : ffff800010130af8: b.gt ffff800010130c04 : if (newpri == oldpri) 0.01 : ffff800010130afc: cmp w3, w2 0.00 : ffff800010130b00: b.eq ffff800010130b9c // b.none 0.09 : ffff800010130b04: mov w4, #0x28 // #40 0.00 : ffff800010130b08: smull x4, w2, w4 0.00 : ffff800010130b0c: add x5, x0, x4 0.00 : ffff800010130b10: add x4, x4, #0x8 : set_bit(): : * See Documentation/atomic_bitops.txt for details. : */ : : static inline void set_bit(unsigned int nr, volatile unsigned long *p) : { : p += BIT_WORD(nr); 0.27 : ffff800010130b14: lsr w6, w1, #6 : atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff800010130b18: mov x7, #0x1 // #1 0.00 : ffff800010130b1c: lsl x1, x7, x1 : p += BIT_WORD(nr); 0.00 : ffff800010130b20: lsl x6, x6, #3 0.00 : ffff800010130b24: add x4, x6, x4 0.00 : ffff800010130b28: add x4, x0, x4 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010130b2c: b ffff800010130bd0 0.00 : ffff800010130b30: b ffff800010130bd0 : __lse_atomic64_or(): : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) 0.01 : ffff800010130b34: mov x7, x1 0.12 : ffff800010130b38: stset x7, [x4] : cpupri_set(): : /* : * When adding a new vector, we update the mask first, : * do a write memory barrier, and then update the count, to : * make sure the vector is visible when count is set. : */ : smp_mb__before_atomic(); 0.49 : ffff800010130b3c: dmb ish : arch_static_branch_jump(): 28.72 : ffff800010130b40: b ffff800010130bbc 0.00 : ffff800010130b44: b ffff800010130bbc : __lse_atomic_add(): : ATOMIC_OP(add, stadd) 0.00 : ffff800010130b48: mov w4, #0x1 // #1 0.00 : ffff800010130b4c: stadd w4, [x5] : cpupri_set(): : atomic_inc(&(vec)->count); : do_mb = 1; : } : if (likely(oldpri != CPUPRI_INVALID)) { 0.12 : ffff800010130b50: cmn w3, #0x1 0.00 : ffff800010130b54: b.eq ffff800010130b98 // b.none : * : * We only need to do a memory barrier if we updated : * the new priority vec. : */ : if (do_mb) : smp_mb__after_atomic(); 0.00 : ffff800010130b58: dmb ish 27.02 : ffff800010130b5c: mov w4, #0x28 // #40 0.00 : ffff800010130b60: smull x3, w3, w4 : : /* : * When removing from the vector, we decrement the counter first : * do a memory barrier and then clear the mask. : */ : atomic_dec(&(vec)->count); 0.00 : ffff800010130b64: add x4, x0, x3 : arch_static_branch_jump(): 0.00 : ffff800010130b68: b ffff800010130be4 0.00 : ffff800010130b6c: b ffff800010130be4 : __lse_atomic_sub(): : asm volatile( 0.00 : ffff800010130b70: mov w5, #0x1 // #1 0.00 : ffff800010130b74: neg w5, w5 0.00 : ffff800010130b78: stadd w5, [x4] : cpupri_set(): : smp_mb__after_atomic(); 0.04 : ffff800010130b7c: dmb ish : clear_bit(): : } : : static inline void clear_bit(unsigned int nr, volatile unsigned long *p) : { : p += BIT_WORD(nr); 28.66 : ffff800010130b80: add x3, x3, #0x8 0.00 : ffff800010130b84: add x3, x3, x6 0.00 : ffff800010130b88: add x0, x0, x3 : arch_static_branch_jump(): 0.00 : ffff800010130b8c: b ffff800010130bd8 0.00 : ffff800010130b90: b ffff800010130bd8 : __lse_atomic64_andnot(): : ATOMIC64_OP(andnot, stclr) 0.00 : ffff800010130b94: stclr x1, [x0] : cpupri_set(): : cpumask_clear_cpu(cpu, vec->mask); : } : : *currpri = newpri; 0.00 : ffff800010130b98: str w2, [x9, x8, lsl #2] : } 0.00 : ffff800010130b9c: ret : if (newpri == oldpri) 0.00 : ffff800010130ba0: cmn w3, #0x1 0.00 : ffff800010130ba4: b.eq ffff800010130b9c // b.none 0.00 : ffff800010130ba8: lsr w6, w1, #6 0.00 : ffff800010130bac: mov x4, #0x1 // #1 0.00 : ffff800010130bb0: lsl x1, x4, x1 0.00 : ffff800010130bb4: lsl x6, x6, #3 0.00 : ffff800010130bb8: b ffff800010130b5c : __ll_sc_atomic_add(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff800010130bbc: b ffff800010130ce8 : cpupri_set(): : if (likely(oldpri != CPUPRI_INVALID)) { 0.00 : ffff800010130bc0: cmn w3, #0x1 0.00 : ffff800010130bc4: b.ne ffff800010130b58 // b.any : *currpri = newpri; 0.00 : ffff800010130bc8: str w2, [x9, x8, lsl #2] 0.00 : ffff800010130bcc: b ffff800010130b9c : __ll_sc_atomic64_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010130bd0: b ffff800010130d00 0.00 : ffff800010130bd4: b ffff800010130b3c : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff800010130bd8: b ffff800010130d18 : cpupri_set(): 0.00 : ffff800010130bdc: str w2, [x9, x8, lsl #2] 0.00 : ffff800010130be0: b ffff800010130b9c : __ll_sc_atomic_sub(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010130be4: mov w5, #0x1 // #1 0.00 : ffff800010130be8: b ffff800010130d30 0.00 : ffff800010130bec: b ffff800010130b7c : cpupri_set(): : if (newpri == oldpri) 0.00 : ffff800010130bf0: cbz w3, ffff800010130b9c 0.00 : ffff800010130bf4: mov x5, x0 0.00 : ffff800010130bf8: mov x4, #0x8 // #8 0.00 : ffff800010130bfc: mov w2, #0x0 // #0 0.00 : ffff800010130c00: b ffff800010130b14 : BUG_ON(newpri >= CPUPRI_NR_PRIORITIES); 0.00 : ffff800010130c04: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (3890 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001046a8d0 : : blk_mq_put_tag(): : return tag + tag_offset; : } : : void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, : unsigned int tag) : { 51.25 : ffff80001046a8d0: stp x29, x30, [sp, #-16]! 0.00 : ffff80001046a8d4: mov w3, w2 0.00 : ffff80001046a8d8: mov x29, sp : if (!blk_mq_tag_is_reserved(tags, tag)) { 0.05 : ffff80001046a8dc: ldr w2, [x0, #4] 4.50 : ffff80001046a8e0: cmp w3, w2 0.05 : ffff80001046a8e4: b.cc ffff80001046a910 // b.lo, b.ul, b.last : const int real_tag = tag - tags->nr_reserved_tags; : : BUG_ON(real_tag >= tags->nr_tags); 29.17 : ffff80001046a8e8: ldr w4, [x0] : const int real_tag = tag - tags->nr_reserved_tags; 0.03 : ffff80001046a8ec: sub w3, w3, w2 : BUG_ON(real_tag >= tags->nr_tags); 0.08 : ffff80001046a8f0: cmp w3, w4 0.00 : ffff80001046a8f4: b.cs ffff80001046a928 // b.hs, b.nlast : sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu); 11.12 : ffff80001046a8f8: ldr w2, [x1, #64] 0.05 : ffff80001046a8fc: add x0, x0, #0x10 0.36 : ffff80001046a900: mov w1, w3 0.00 : ffff80001046a904: bl ffff8000104b1058 : } else { : BUG_ON(tag >= tags->nr_reserved_tags); : sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu); : } : } 3.35 : ffff80001046a908: ldp x29, x30, [sp], #16 0.00 : ffff80001046a90c: ret : sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu); 0.00 : ffff80001046a910: ldr w2, [x1, #64] 0.00 : ffff80001046a914: add x0, x0, #0x50 0.00 : ffff80001046a918: mov w1, w3 0.00 : ffff80001046a91c: bl ffff8000104b1058 : } 0.00 : ffff80001046a920: ldp x29, x30, [sp], #16 0.00 : ffff80001046a924: ret : BUG_ON(real_tag >= tags->nr_tags); 0.00 : ffff80001046a928: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (3943 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010114bb8 : : resched_curr(): : * might also involve a cross-CPU call to trigger the scheduler on : * the target CPU. : */ : void resched_curr(struct rq *rq) : { : struct task_struct *curr = rq->curr; 2.29 : ffff800010114bb8: ldr x1, [x0, #2352] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.62 : ffff800010114bbc: ldr x2, [x1] : resched_curr(): : int cpu; : : lockdep_assert_held(&rq->lock); : : if (test_tsk_need_resched(curr)) 0.03 : ffff800010114bc0: tst w2, #0x2 0.00 : ffff800010114bc4: b.ne ffff800010114c38 // b.any : { 62.20 : ffff800010114bc8: stp x29, x30, [sp, #-16]! : return; : : cpu = cpu_of(rq); : : if (cpu == smp_processor_id()) { 0.00 : ffff800010114bcc: adrp x2, ffff80001151d000 0.00 : ffff800010114bd0: add x2, x2, #0x18 : { 0.00 : ffff800010114bd4: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 5.78 : ffff800010114bd8: mrs x3, tpidr_el1 : resched_curr(): : cpu = cpu_of(rq); 0.51 : ffff800010114bdc: ldr w0, [x0, #2568] : if (cpu == smp_processor_id()) { 0.00 : ffff800010114be0: ldr w2, [x2, x3] 0.00 : ffff800010114be4: cmp w2, w0 0.00 : ffff800010114be8: b.eq ffff800010114c10 // b.none : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 6.12 : ffff800010114bec: b ffff800010114c08 6.02 : ffff800010114bf0: b ffff800010114c08 : __lse_atomic64_or(): : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) 0.25 : ffff800010114bf4: mov x2, #0x2 // #2 0.02 : ffff800010114bf8: stset x2, [x1] : resched_curr(): : set_preempt_need_resched(); : return; : } : : if (set_nr_and_not_polling(curr)) : smp_send_reschedule(cpu); 8.69 : ffff800010114bfc: bl ffff800010096ae0 : else : trace_sched_wake_idle_without_ipi(cpu); : } 6.11 : ffff800010114c00: ldp x29, x30, [sp], #16 0.00 : ffff800010114c04: ret : __ll_sc_atomic64_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010114c08: b ffff80001011aa9c 0.00 : ffff800010114c0c: b ffff800010114bfc : arch_static_branch_jump(): 0.19 : ffff800010114c10: b ffff800010114c24 0.10 : ffff800010114c14: b ffff800010114c24 : __lse_atomic64_or(): 0.00 : ffff800010114c18: mov x0, #0x2 // #2 0.00 : ffff800010114c1c: stset x0, [x1] 0.96 : ffff800010114c20: b ffff800010114c28 : __ll_sc_atomic64_or(): 0.00 : ffff800010114c24: b ffff80001011aab4 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010114c28: mrs x0, sp_el0 : set_preempt_need_resched(): : task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \ : } while (0) : : static inline void set_preempt_need_resched(void) : { : current_thread_info()->preempt.need_resched = 0; 0.00 : ffff800010114c2c: str wzr, [x0, #20] : resched_curr(): 0.12 : ffff800010114c30: ldp x29, x30, [sp], #16 0.00 : ffff800010114c34: ret 0.00 : ffff800010114c38: ret Percent | Source code & Disassembly of vmlinux for cycles (3941 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015c1b0 <__rcu_read_lock>: : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 40.55 : ffff80001015c1b0: mrs x1, sp_el0 : rcu_preempt_read_enter(): : #define RCU_NEST_NMAX (-INT_MAX / 2) : #define RCU_NEST_PMAX (INT_MAX / 2) : : static void rcu_preempt_read_enter(void) : { : current->rcu_read_lock_nesting++; 18.62 : ffff80001015c1b4: ldr w0, [x1, #776] 0.00 : ffff80001015c1b8: add w0, w0, #0x1 40.79 : ffff80001015c1bc: str w0, [x1, #776] : __rcu_read_lock(): : { : rcu_preempt_read_enter(); : if (IS_ENABLED(CONFIG_PROVE_LOCKING)) : WARN_ON_ONCE(rcu_preempt_depth() > RCU_NEST_PMAX); : barrier(); /* critical section after entry code. */ : } 0.05 : ffff80001015c1c0: ret Percent | Source code & Disassembly of vmlinux for cycles (3755 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011fd78 : : update_cfs_group(): : * Recomputes the group entity based on the current state of its group : * runqueue. : */ : static void update_cfs_group(struct sched_entity *se) : { : struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3.71 : ffff80001011fd78: ldr x7, [x0, #136] : long shares, runnable; : : if (!gcfs_rq) 0.00 : ffff80001011fd7c: cbz x7, ffff80001011fe20 : { 0.55 : ffff80001011fd80: stp x29, x30, [sp, #-16]! 0.00 : ffff80001011fd84: mov x29, sp : calc_group_shares(): : load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 5.37 : ffff80001011fd88: ldr x6, [x7] : struct task_group *tg = cfs_rq->tg; 2.36 : ffff80001011fd8c: ldr x4, [x7, #336] : load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 6.19 : ffff80001011fd90: ldr x2, [x7, #160] 0.00 : ffff80001011fd94: lsr x6, x6, #10 : tg_weight -= cfs_rq->tg_load_avg_contrib; 2.06 : ffff80001011fd98: ldr x5, [x7, #256] : load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 0.00 : ffff80001011fd9c: cmp x6, x2 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.32 : ffff80001011fda0: ldr x3, [x4, #256] 25.20 : ffff80001011fda4: ldr x1, [x4, #320] : calc_group_shares(): 0.00 : ffff80001011fda8: csel x6, x6, x2, cs // cs = hs, nlast : tg_weight -= cfs_rq->tg_load_avg_contrib; 0.13 : ffff80001011fdac: sub x1, x1, x5 : if (tg_weight) 0.00 : ffff80001011fdb0: adds x1, x6, x1 : shares = (tg_shares * load); 20.91 : ffff80001011fdb4: mul x5, x3, x6 : if (tg_weight) 0.03 : ffff80001011fdb8: b.eq ffff80001011fdc0 // b.none : shares /= tg_weight; 0.05 : ffff80001011fdbc: sdiv x5, x5, x1 : calc_group_runnable(): : runnable = max(cfs_rq->avg.runnable_load_avg, 9.01 : ffff80001011fdc0: ldr x2, [x7, #16] : calc_group_shares(): : return clamp_t(long, shares, MIN_SHARES, tg_shares); 0.00 : ffff80001011fdc4: cmp x5, #0x2 : calc_group_runnable(): : runnable = max(cfs_rq->avg.runnable_load_avg, 0.80 : ffff80001011fdc8: ldr x1, [x7, #168] : calc_group_shares(): : return clamp_t(long, shares, MIN_SHARES, tg_shares); 0.00 : ffff80001011fdcc: mov x4, #0x2 // #2 0.00 : ffff80001011fdd0: csel x4, x5, x4, ge // ge = tcont 0.00 : ffff80001011fdd4: cmp x4, x3 : calc_group_runnable(): : runnable = max(cfs_rq->avg.runnable_load_avg, 1.84 : ffff80001011fdd8: lsr x2, x2, #10 : calc_group_shares(): : return clamp_t(long, shares, MIN_SHARES, tg_shares); 0.00 : ffff80001011fddc: csel x4, x4, x3, le : calc_group_runnable(): : runnable = max(cfs_rq->avg.runnable_load_avg, 0.00 : ffff80001011fde0: cmp x2, x1 0.00 : ffff80001011fde4: csel x2, x2, x1, cs // cs = hs, nlast : runnable *= shares; 0.00 : ffff80001011fde8: mul x2, x2, x4 : if (load_avg) 0.00 : ffff80001011fdec: cbz x6, ffff80001011fdf4 : runnable /= load_avg; 4.16 : ffff80001011fdf0: sdiv x2, x2, x6 : return clamp_t(long, runnable, MIN_SHARES, shares); 0.00 : ffff80001011fdf4: cmp x2, #0x2 0.00 : ffff80001011fdf8: mov x1, x0 0.05 : ffff80001011fdfc: mov x3, #0x2 // #2 : update_cfs_group(): : #else : shares = calc_group_shares(gcfs_rq); : runnable = calc_group_runnable(gcfs_rq, shares); : #endif : : reweight_entity(cfs_rq_of(se), se, shares, runnable); 0.67 : ffff80001011fe00: ldr x0, [x0, #128] : calc_group_runnable(): : return clamp_t(long, runnable, MIN_SHARES, shares); 0.00 : ffff80001011fe04: csel x3, x2, x3, ge // ge = tcont : update_cfs_group(): : reweight_entity(cfs_rq_of(se), se, shares, runnable); 0.00 : ffff80001011fe08: mov x2, x4 : calc_group_runnable(): : return clamp_t(long, runnable, MIN_SHARES, shares); 0.00 : ffff80001011fe0c: cmp x3, x4 : update_cfs_group(): : reweight_entity(cfs_rq_of(se), se, shares, runnable); 3.87 : ffff80001011fe10: csel x3, x3, x4, le 0.00 : ffff80001011fe14: bl ffff80001011fbf8 : } 3.01 : ffff80001011fe18: ldp x29, x30, [sp], #16 0.08 : ffff80001011fe1c: ret 8.62 : ffff80001011fe20: ret Percent | Source code & Disassembly of vmlinux for cycles (3837 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010118820 : : idle_cpu(): : * : * Return: 1 if the CPU is currently idle. 0 otherwise. : */ : int idle_cpu(int cpu) : { : struct rq *rq = cpu_rq(cpu); 5.57 : ffff800010118820: adrp x2, ffff800011909000 0.00 : ffff800010118824: add x2, x2, #0x928 0.00 : ffff800010118828: adrp x1, ffff800011528000 0.00 : ffff80001011882c: add x1, x1, #0xe80 0.00 : ffff800010118830: ldr x0, [x2, w0, sxtw #3] 0.02 : ffff800010118834: add x1, x1, x0 : : if (rq->curr != rq->idle) 46.44 : ffff800010118838: ldr x2, [x1, #2352] 39.23 : ffff80001011883c: ldr x0, [x1, #2360] 0.00 : ffff800010118840: cmp x2, x0 0.00 : ffff800010118844: b.eq ffff800010118850 // b.none : return 0; 7.12 : ffff800010118848: mov w0, #0x0 // #0 : if (!llist_empty(&rq->wake_list)) : return 0; : #endif : : return 1; : } 0.60 : ffff80001011884c: ret : if (rq->nr_running) 0.17 : ffff800010118850: ldr w2, [x1, #4] : return 0; 0.00 : ffff800010118854: mov w0, #0x0 // #0 : if (rq->nr_running) 0.02 : ffff800010118858: cbnz w2, ffff80001011884c : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.82 : ffff80001011885c: ldr x0, [x1, #2976] : idle_cpu(): : if (!llist_empty(&rq->wake_list)) 0.02 : ffff800010118860: cmp x0, #0x0 0.00 : ffff800010118864: cset w0, eq // eq = none : } 0.00 : ffff800010118868: ret Percent | Source code & Disassembly of vmlinux for cycles (3686 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001014db58 : : irq_thread(): : : /* : * Interrupt handler thread : */ : static int irq_thread(void *data) : { 0.00 : ffff80001014db58: stp x29, x30, [sp, #-128]! 0.00 : ffff80001014db5c: mov x29, sp 0.00 : ffff80001014db60: stp x21, x22, [sp, #32] 0.00 : ffff80001014db64: mov x21, x0 0.00 : ffff80001014db68: stp x19, x20, [sp, #16] 0.00 : ffff80001014db6c: stp x23, x24, [sp, #48] 0.00 : ffff80001014db70: stp x25, x26, [sp, #64] 0.00 : ffff80001014db74: adrp x25, ffff800011909000 0.00 : ffff80001014db78: stp x27, x28, [sp, #80] 0.00 : ffff80001014db7c: add x0, x25, #0x908 0.00 : ffff80001014db80: ldr x1, [x0] 0.00 : ffff80001014db84: str x1, [x29, #120] 0.00 : ffff80001014db88: mov x1, #0x0 // #0 : struct callback_head on_exit_work; : struct irqaction *action = data; : struct irq_desc *desc = irq_to_desc(action->irq); 0.00 : ffff80001014db8c: ldr w0, [x21, #56] 0.00 : ffff80001014db90: bl ffff80001014b3a8 0.00 : ffff80001014db94: mov x22, x0 : irqreturn_t (*handler_fn)(struct irq_desc *desc, : struct irqaction *action); : : if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD, 0.00 : ffff80001014db98: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001014db9c: ldrb w0, [x0, #640] 0.00 : ffff80001014dba0: cbz w0, ffff80001014dd58 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001014dba4: ldr x1, [x21, #64] : irq_thread(): : &action->thread_flags)) : handler_fn = irq_forced_thread_fn; 0.00 : ffff80001014dba8: adrp x0, ffff80001014d000 <__free_percpu_irq+0x88> 0.00 : ffff80001014dbac: adrp x24, ffff80001014d000 <__free_percpu_irq+0x88> 0.00 : ffff80001014dbb0: add x0, x0, #0xf38 0.00 : ffff80001014dbb4: add x24, x24, #0xda0 0.00 : ffff80001014dbb8: tst w1, #0x8 0.00 : ffff80001014dbbc: csel x24, x24, x0, eq // eq = none : init_task_work(): : typedef void (*task_work_func_t)(struct callback_head *); : : static inline void : init_task_work(struct callback_head *twork, task_work_func_t func) : { : twork->func = func; 0.00 : ffff80001014dbc0: adrp x26, ffff80001014d000 <__free_percpu_irq+0x88> : irq_thread(): : else : handler_fn = irq_thread_fn; : : init_task_work(&on_exit_work, irq_thread_dtor); : task_work_add(current, &on_exit_work, false); 0.00 : ffff80001014dbc4: mov w2, #0x0 // #0 : init_task_work(): 0.00 : ffff80001014dbc8: add x0, x26, #0xe38 : irq_thread(): 0.00 : ffff80001014dbcc: add x1, x29, #0x68 : init_task_work(): 0.00 : ffff80001014dbd0: str x0, [x29, #112] : irq_thread_check_affinity(): : if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags)) 0.00 : ffff80001014dbd4: add x19, x21, #0x40 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001014dbd8: mrs x0, sp_el0 : irq_thread(): : task_work_add(current, &on_exit_work, false); 0.00 : ffff80001014dbdc: bl ffff80001010a180 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001014dbe0: ldr x0, [x21, #64] : test_and_clear_bit(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (!(READ_ONCE(*p) & mask)) 0.00 : ffff80001014dbe4: tbnz w0, #2, ffff80001014dd64 : irq_wake_secondary(): : raw_spin_lock_irq(&desc->lock); 0.00 : ffff80001014dbe8: add x27, x22, #0xd4 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001014dbec: mov x20, #0x1 // #1 : __ll_sc_atomic64_fetch_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff80001014dbf0: mov x23, #0x4 // #4 0.00 : ffff80001014dbf4: nop : get_current(): 0.33 : ffff80001014dbf8: mrs x0, sp_el0 : __write_once_size(): 0.03 : ffff80001014dbfc: str x20, [x0, #24] : irq_wait_for_interrupt(): : set_current_state(TASK_INTERRUPTIBLE); 1.06 : ffff80001014dc00: dmb ish : if (kthread_should_stop()) { 16.38 : ffff80001014dc04: bl ffff80001010bc70 0.14 : ffff80001014dc08: tst w0, #0xff : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001014dc0c: ldr x0, [x21, #64] : irq_wait_for_interrupt(): 0.00 : ffff80001014dc10: b.ne ffff80001014dc3c // b.any 0.06 : ffff80001014dc14: nop : test_and_clear_bit(): 1.05 : ffff80001014dc18: tbnz w0, #0, ffff80001014dc84 : irq_wait_for_interrupt(): : schedule(); 0.00 : ffff80001014dc1c: bl ffff800010cd2a78 : get_current(): 1.17 : ffff80001014dc20: mrs x0, sp_el0 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.24 : ffff80001014dc24: str x20, [x0, #24] : irq_wait_for_interrupt(): : set_current_state(TASK_INTERRUPTIBLE); 2.34 : ffff80001014dc28: dmb ish : if (kthread_should_stop()) { 1.63 : ffff80001014dc2c: bl ffff80001010bc70 1.30 : ffff80001014dc30: tst w0, #0xff : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001014dc34: ldr x0, [x21, #64] : irq_wait_for_interrupt(): 0.00 : ffff80001014dc38: b.eq ffff80001014dc18 // b.none : test_and_clear_bit(): 0.00 : ffff80001014dc3c: tbnz w0, #0, ffff80001014dd34 : get_current(): 0.00 : ffff80001014dc40: mrs x0, sp_el0 : irq_wait_for_interrupt(): : __set_current_state(TASK_RUNNING); 0.00 : ffff80001014dc44: str xzr, [x0, #24] : irq_thread(): : * This is the regular exit path. __free_irq() is stopping the : * thread via kthread_stop() after calling : * synchronize_hardirq(). So neither IRQTF_RUNTHREAD nor the : * oneshot mask bit can be set. : */ : task_work_cancel(current, irq_thread_dtor); 0.00 : ffff80001014dc48: add x1, x26, #0xe38 : return 0; : } 0.00 : ffff80001014dc4c: add x25, x25, #0x908 : task_work_cancel(current, irq_thread_dtor); 0.00 : ffff80001014dc50: bl ffff80001010a270 : } 0.00 : ffff80001014dc54: mov w0, #0x0 // #0 0.00 : ffff80001014dc58: ldr x2, [x29, #120] 0.00 : ffff80001014dc5c: ldr x1, [x25] 0.00 : ffff80001014dc60: eor x1, x2, x1 0.00 : ffff80001014dc64: cbnz x1, ffff80001014dd9c 0.00 : ffff80001014dc68: ldp x19, x20, [sp, #16] 0.00 : ffff80001014dc6c: ldp x21, x22, [sp, #32] 0.00 : ffff80001014dc70: ldp x23, x24, [sp, #48] 0.00 : ffff80001014dc74: ldp x25, x26, [sp, #64] 0.00 : ffff80001014dc78: ldp x27, x28, [sp, #80] 0.00 : ffff80001014dc7c: ldp x29, x30, [sp], #128 0.00 : ffff80001014dc80: ret : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.37 : ffff80001014dc84: b ffff80001014dcc8 0.00 : ffff80001014dc88: b ffff80001014dcc8 : __lse_atomic64_fetch_andnot(): : ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \ : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) 0.00 : ffff80001014dc8c: mov x0, x20 0.84 : ffff80001014dc90: ldclral x0, x0, [x19] : irq_wait_for_interrupt(): : if (test_and_clear_bit(IRQTF_RUNTHREAD, 38.58 : ffff80001014dc94: tbz w0, #0, ffff80001014dc1c : get_current(): 0.00 : ffff80001014dc98: mrs x0, sp_el0 : irq_wait_for_interrupt(): : __set_current_state(TASK_RUNNING); 1.41 : ffff80001014dc9c: str xzr, [x0, #24] : __read_once_size(): 14.45 : ffff80001014dca0: ldr x0, [x21, #64] : test_and_clear_bit(): 0.00 : ffff80001014dca4: tbnz w0, #2, ffff80001014dcd4 : irq_thread(): : action_ret = handler_fn(desc, action); 13.90 : ffff80001014dca8: mov x1, x21 0.00 : ffff80001014dcac: mov x0, x22 0.00 : ffff80001014dcb0: blr x24 : if (action_ret == IRQ_WAKE_THREAD) 0.05 : ffff80001014dcb4: cmp w0, #0x2 0.00 : ffff80001014dcb8: b.eq ffff80001014dd04 // b.none : wake_threads_waitq(desc); 0.58 : ffff80001014dcbc: mov x0, x22 0.03 : ffff80001014dcc0: bl ffff80001014d4e8 4.07 : ffff80001014dcc4: b ffff80001014dbf8 : __ll_sc_atomic64_fetch_andnot(): 0.00 : ffff80001014dcc8: b ffff800010150010 <__irq_get_irqchip_state+0xb0> : irq_wait_for_interrupt(): : if (test_and_clear_bit(IRQTF_RUNTHREAD, 0.00 : ffff80001014dccc: tbz w0, #0, ffff80001014dc1c 0.00 : ffff80001014dcd0: b ffff80001014dc98 : arch_static_branch_jump(): 0.00 : ffff80001014dcd4: b ffff80001014dd2c 0.00 : ffff80001014dcd8: b ffff80001014dd2c : __lse_atomic64_fetch_andnot(): 0.00 : ffff80001014dcdc: mov x0, x23 0.00 : ffff80001014dce0: ldclral x0, x0, [x19] : irq_thread_check_affinity(): : if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags)) 0.00 : ffff80001014dce4: tbz w0, #2, ffff80001014dca8 0.00 : ffff80001014dce8: mov x0, x22 0.00 : ffff80001014dcec: bl ffff80001014d328 : irq_thread(): : action_ret = handler_fn(desc, action); 0.00 : ffff80001014dcf0: mov x1, x21 0.00 : ffff80001014dcf4: mov x0, x22 0.00 : ffff80001014dcf8: blr x24 : if (action_ret == IRQ_WAKE_THREAD) 0.00 : ffff80001014dcfc: cmp w0, #0x2 0.00 : ffff80001014dd00: b.ne ffff80001014dcbc // b.any : irq_wake_secondary(desc, action); 0.00 : ffff80001014dd04: ldr x28, [x21, #48] : irq_wake_secondary(): : if (WARN_ON_ONCE(!secondary)) 0.00 : ffff80001014dd08: cbz x28, ffff80001014dd94 : raw_spin_lock_irq(&desc->lock); 0.00 : ffff80001014dd0c: mov x0, x27 0.00 : ffff80001014dd10: bl ffff800010cd8460 <_raw_spin_lock_irq> : __irq_wake_thread(desc, secondary); 0.00 : ffff80001014dd14: mov x1, x28 0.00 : ffff80001014dd18: mov x0, x22 0.00 : ffff80001014dd1c: bl ffff80001014c358 <__irq_wake_thread> : raw_spin_unlock_irq(&desc->lock); 0.00 : ffff80001014dd20: mov x0, x27 0.00 : ffff80001014dd24: bl ffff800010cd7c70 <_raw_spin_unlock_irq> 0.00 : ffff80001014dd28: b ffff80001014dcbc : __ll_sc_atomic64_fetch_andnot(): 0.00 : ffff80001014dd2c: b ffff80001015002c <__irq_get_irqchip_state+0xcc> 0.00 : ffff80001014dd30: b ffff80001014dce4 : arch_static_branch_jump(): 0.00 : ffff80001014dd34: b ffff80001014dd4c 0.00 : ffff80001014dd38: b ffff80001014dd4c : __lse_atomic64_fetch_andnot(): 0.00 : ffff80001014dd3c: mov x0, x20 0.00 : ffff80001014dd40: ldclral x0, x0, [x19] : irq_wait_for_interrupt(): : if (test_and_clear_bit(IRQTF_RUNTHREAD, 0.00 : ffff80001014dd44: tbnz w0, #0, ffff80001014dc98 0.00 : ffff80001014dd48: b ffff80001014dc40 : __ll_sc_atomic64_fetch_andnot(): 0.00 : ffff80001014dd4c: b ffff800010150048 <__irq_get_irqchip_state+0xe8> : irq_wait_for_interrupt(): 0.00 : ffff80001014dd50: tbnz w0, #0, ffff80001014dc98 0.00 : ffff80001014dd54: b ffff80001014dc40 : irq_thread(): : handler_fn = irq_thread_fn; 0.00 : ffff80001014dd58: adrp x24, ffff80001014d000 <__free_percpu_irq+0x88> 0.00 : ffff80001014dd5c: add x24, x24, #0xda0 0.00 : ffff80001014dd60: b ffff80001014dbc0 : arch_static_branch_jump(): 0.00 : ffff80001014dd64: b ffff80001014dd84 0.00 : ffff80001014dd68: b ffff80001014dd84 : __lse_atomic64_fetch_andnot(): 0.00 : ffff80001014dd6c: mov x0, #0x4 // #4 0.00 : ffff80001014dd70: ldclral x0, x0, [x19] : irq_thread_check_affinity(): : if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags)) 0.00 : ffff80001014dd74: tbz w0, #2, ffff80001014dbe8 0.00 : ffff80001014dd78: mov x0, x22 0.00 : ffff80001014dd7c: bl ffff80001014d328 0.00 : ffff80001014dd80: b ffff80001014dbe8 : __ll_sc_atomic64_fetch_andnot(): 0.00 : ffff80001014dd84: mov x1, #0x4 // #4 0.00 : ffff80001014dd88: add x4, x21, #0x40 0.00 : ffff80001014dd8c: b ffff800010150064 <__irq_get_irqchip_state+0x104> 0.00 : ffff80001014dd90: b ffff80001014dd74 : irq_wake_secondary(): : if (WARN_ON_ONCE(!secondary)) 0.00 : ffff80001014dd94: brk #0x800 0.00 : ffff80001014dd98: b ffff80001014dcbc : irq_thread(): : } 0.00 : ffff80001014dd9c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (3522 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001014d4e8 : : wake_threads_waitq(): : irq_finalize_oneshot(desc, action); : return ret; : } : : static void wake_threads_waitq(struct irq_desc *desc) : { 0.48 : ffff80001014d4e8: stp x29, x30, [sp, #-16]! 0.00 : ffff80001014d4ec: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001014d4f0: b ffff80001014d52c 0.63 : ffff80001014d4f4: b ffff80001014d52c 0.00 : ffff80001014d4f8: add x2, x0, #0x100 : __lse_atomic_sub_return(): : } : : ATOMIC_OP_SUB_RETURN(_relaxed, ) : ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC_OP_SUB_RETURN(_release, l, "memory") : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001014d4fc: mov w1, #0x1 // #1 0.00 : ffff80001014d500: neg w1, w1 0.67 : ffff80001014d504: ldaddal w1, w3, [x2] 93.92 : ffff80001014d508: add w1, w1, w3 : wake_threads_waitq(): : if (atomic_dec_and_test(&desc->threads_active)) 0.00 : ffff80001014d50c: cbnz w1, ffff80001014d524 : wake_up(&desc->wait_for_threads); 0.93 : ffff80001014d510: mov x3, #0x0 // #0 0.00 : ffff80001014d514: mov w2, #0x1 // #1 0.00 : ffff80001014d518: mov w1, #0x3 // #3 0.00 : ffff80001014d51c: add x0, x0, #0x108 0.13 : ffff80001014d520: bl ffff80001012f9c0 <__wake_up> : } 3.24 : ffff80001014d524: ldp x29, x30, [sp], #16 0.00 : ffff80001014d528: ret : __ll_sc_atomic_sub_return(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff80001014d52c: mov w2, #0x1 // #1 0.00 : ffff80001014d530: add x4, x0, #0x100 0.00 : ffff80001014d534: b ffff80001014ff94 <__irq_get_irqchip_state+0x34> 0.00 : ffff80001014d538: b ffff80001014d50c Percent | Source code & Disassembly of vmlinux for cycles (3430 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101505c8 : : note_interrupt(): : : void note_interrupt(struct irq_desc *desc, irqreturn_t action_ret) : { : unsigned int irq; : : if (desc->istate & IRQS_POLL_INPROGRESS || 4.79 : ffff8000101505c8: ldr w2, [x0, #172] 0.00 : ffff8000101505cc: tbnz w2, #3, ffff8000101506d0 : { 0.49 : ffff8000101505d0: stp x29, x30, [sp, #-96]! 0.00 : ffff8000101505d4: mov x29, sp 6.02 : ffff8000101505d8: str x19, [sp, #16] 0.00 : ffff8000101505dc: mov x19, x0 : irq_settings_is_polled(): : return desc->status_use_accessors & _IRQ_NESTED_THREAD; : } : : static inline bool irq_settings_is_polled(struct irq_desc *desc) : { : return desc->status_use_accessors & _IRQ_IS_POLLED; 2.21 : ffff8000101505e0: ldr w2, [x0, #168] : note_interrupt(): : if (desc->istate & IRQS_POLL_INPROGRESS || 0.00 : ffff8000101505e4: tbnz w2, #18, ffff8000101506c4 0.73 : ffff8000101505e8: str x20, [x29, #24] : bad_action_ret(): : if (likely(r <= (IRQ_HANDLED | IRQ_WAKE_THREAD))) 0.00 : ffff8000101505ec: cmp w1, #0x3 0.00 : ffff8000101505f0: mov w20, w1 0.00 : ffff8000101505f4: b.hi ffff8000101507a0 // b.pmore 4.74 : ffff8000101505f8: str x22, [x29, #40] : note_interrupt(): : * We could handle all interrupts with the delayed by one : * mechanism, but for the non forced threaded case we'd just : * add pointless overhead to the straight hardirq interrupts : * for the sake of a few lines less code. : */ : if (action_ret & IRQ_WAKE_THREAD) { 0.00 : ffff8000101505fc: tbnz w20, #1, ffff8000101506d4 : */ : desc->threads_handled_last &= ~SPURIOUS_DEFERRED; : } : } : : if (unlikely(action_ret == IRQ_NONE)) { 0.00 : ffff800010150600: cbz w1, ffff8000101507bc : try_misrouted_irq(): : if (!irqfixup) 0.35 : ffff800010150604: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> : note_interrupt(): : else : desc->irqs_unhandled++; : desc->last_unhandled = jiffies; : } : : irq = irq_desc_get_irq(desc); 0.08 : ffff800010150608: ldr w22, [x19, #92] : try_misrouted_irq(): : if (irqfixup < 2) 1.92 : ffff80001015060c: ldr w0, [x0, #648] 0.00 : ffff800010150610: cmp w0, #0x1 0.00 : ffff800010150614: b.le ffff8000101506a0 : if (!irq) 0.00 : ffff800010150618: cbz w22, ffff8000101506f0 : action = desc->action; 0.00 : ffff80001015061c: ldr x0, [x19, #160] : return action && (action->flags & IRQF_IRQPOLL); 0.00 : ffff800010150620: cbz x0, ffff8000101506a0 0.00 : ffff800010150624: ldr w0, [x0, #60] 0.00 : ffff800010150628: tbz w0, #12, ffff8000101506a0 0.00 : ffff80001015062c: stp x23, x24, [x29, #48] 0.00 : ffff800010150630: mov w24, w22 0.00 : ffff800010150634: str x21, [x29, #32] 0.00 : ffff800010150638: str x26, [x29, #72] 0.00 : ffff80001015063c: nop : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010150640: b ffff80001015083c 0.00 : ffff800010150644: b ffff80001015083c : __lse_atomic_add_return(): : } : : ATOMIC_OP_ADD_RETURN(_relaxed, ) : ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") : ATOMIC_OP_ADD_RETURN(_release, l, "memory") : ATOMIC_OP_ADD_RETURN( , al, "memory") 0.00 : ffff800010150648: adrp x21, ffff800011b20000 <__log_buf+0x1fc10> 0.00 : ffff80001015064c: mov w23, #0x1 // #1 0.00 : ffff800010150650: add x0, x21, #0x880 0.00 : ffff800010150654: ldaddal w23, w1, [x0] 0.00 : ffff800010150658: add w23, w23, w1 : misrouted_irq(): : if (atomic_inc_return(&irq_poll_active) != 1) 0.00 : ffff80001015065c: cmp w23, #0x1 0.00 : ffff800010150660: b.eq ffff800010150704 // b.none : int i, ok = 0; 0.00 : ffff800010150664: mov w26, #0x0 // #0 : arch_static_branch_jump(): 0.00 : ffff800010150668: b ffff80001015084c 0.00 : ffff80001015066c: b ffff80001015084c : __lse_atomic_sub(): : : #undef ATOMIC_FETCH_OP_AND : : static inline void __lse_atomic_sub(int i, atomic_t *v) : { : asm volatile( 0.00 : ffff800010150670: mov w0, #0x1 // #1 0.00 : ffff800010150674: add x1, x21, #0x880 0.00 : ffff800010150678: neg w0, w0 0.00 : ffff80001015067c: stadd w0, [x1] : note_interrupt(): : if (unlikely(try_misrouted_irq(irq, desc, action_ret))) { : int ok = misrouted_irq(irq); : if (action_ret == IRQ_NONE) 0.00 : ffff800010150680: cbnz w20, ffff80001015088c : desc->irqs_unhandled -= ok; 0.00 : ffff800010150684: ldr w0, [x19, #200] 0.00 : ffff800010150688: sub w26, w0, w26 0.00 : ffff80001015068c: str w26, [x19, #200] 0.00 : ffff800010150690: ldr x21, [x29, #32] 0.00 : ffff800010150694: ldp x23, x24, [x29, #48] 0.00 : ffff800010150698: ldr x26, [x29, #72] 0.00 : ffff80001015069c: nop : } : : desc->irq_count++; 6.74 : ffff8000101506a0: ldr w0, [x19, #188] : if (likely(desc->irq_count < 100000)) 0.00 : ffff8000101506a4: mov w1, #0x869f // #34463 0.00 : ffff8000101506a8: movk w1, #0x1, lsl #16 : desc->irq_count++; 0.00 : ffff8000101506ac: add w0, w0, #0x1 7.31 : ffff8000101506b0: str w0, [x19, #188] : if (likely(desc->irq_count < 100000)) 0.00 : ffff8000101506b4: cmp w0, w1 0.00 : ffff8000101506b8: b.hi ffff800010150814 // b.pmore 2.54 : ffff8000101506bc: ldr x20, [x29, #24] 11.25 : ffff8000101506c0: ldr x22, [x29, #40] : : mod_timer(&poll_spurious_irq_timer, : jiffies + POLL_SPURIOUS_IRQ_INTERVAL); : } : desc->irqs_unhandled = 0; : } 2.64 : ffff8000101506c4: ldr x19, [sp, #16] 0.70 : ffff8000101506c8: ldp x29, x30, [sp], #96 0.00 : ffff8000101506cc: ret 0.00 : ffff8000101506d0: ret : if (action_ret == IRQ_WAKE_THREAD) { 1.09 : ffff8000101506d4: cmp w1, #0x2 2.45 : ffff8000101506d8: ldr w0, [x0, #208] 0.00 : ffff8000101506dc: b.eq ffff800010150788 // b.none : desc->threads_handled_last &= ~SPURIOUS_DEFERRED; 0.00 : ffff8000101506e0: and w0, w0, #0x7fffffff 0.00 : ffff8000101506e4: mov w20, #0x3 // #3 0.00 : ffff8000101506e8: str w0, [x19, #208] 0.00 : ffff8000101506ec: b ffff800010150604 0.00 : ffff8000101506f0: str x21, [x29, #32] 0.00 : ffff8000101506f4: stp x23, x24, [x29, #48] : try_misrouted_irq(): : if (!irq) 0.00 : ffff8000101506f8: mov w24, #0x0 // #0 0.00 : ffff8000101506fc: str x26, [x29, #72] 0.00 : ffff800010150700: b ffff800010150640 0.00 : ffff800010150704: str x25, [x29, #64] : misrouted_irq(): : irq_poll_cpu = smp_processor_id(); 0.00 : ffff800010150708: adrp x1, ffff80001151d000 0.00 : ffff80001015070c: str x27, [x29, #80] 0.00 : ffff800010150710: add x1, x1, #0x18 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010150714: mrs x3, tpidr_el1 : misrouted_irq(): 0.00 : ffff800010150718: ldr w1, [x1, x3] 0.00 : ffff80001015071c: add x2, x21, #0x880 : for_each_irq_desc(i, desc) { 0.00 : ffff800010150720: mov w0, #0x0 // #0 : int i, ok = 0; 0.00 : ffff800010150724: mov w26, #0x0 // #0 : for_each_irq_desc(i, desc) { 0.00 : ffff800010150728: mov w25, #0x0 // #0 : irq_poll_cpu = smp_processor_id(); 0.00 : ffff80001015072c: str w1, [x2, #4] : for_each_irq_desc(i, desc) { 0.00 : ffff800010150730: bl ffff80001014b3a8 0.00 : ffff800010150734: adrp x0, ffff800011921000 0.00 : ffff800010150738: add x27, x0, #0xc18 0.00 : ffff80001015073c: ldr w0, [x0, #3096] 0.00 : ffff800010150740: cmp w0, #0x0 0.00 : ffff800010150744: b.le ffff800010150904 0.00 : ffff800010150748: add w25, w25, #0x1 0.00 : ffff80001015074c: mov w0, w25 0.00 : ffff800010150750: bl ffff80001014b3a8 0.00 : ffff800010150754: ldr w1, [x27] 0.00 : ffff800010150758: mov x2, x0 0.00 : ffff80001015075c: cmp w25, w1 0.00 : ffff800010150760: b.ge ffff8000101508a8 // b.tcont : if (i == irq) /* Already tried */ 0.00 : ffff800010150764: cmp w25, #0x0 0.00 : ffff800010150768: ccmp w25, w24, #0x4, ne // ne = any 0.00 : ffff80001015076c: ccmp x2, #0x0, #0x4, ne // ne = any 0.00 : ffff800010150770: b.eq ffff800010150748 // b.none : if (try_one_irq(desc, false)) 0.00 : ffff800010150774: mov w1, #0x0 // #0 0.00 : ffff800010150778: bl ffff8000101501d8 : ok = 1; 0.00 : ffff80001015077c: cmp w0, #0x0 0.00 : ffff800010150780: csel w26, w26, w23, eq // eq = none 0.00 : ffff800010150784: b ffff800010150748 : note_interrupt(): : if (!(desc->threads_handled_last & SPURIOUS_DEFERRED)) { 1.20 : ffff800010150788: tbnz w0, #31, ffff80001015085c : desc->threads_handled_last |= SPURIOUS_DEFERRED; 0.00 : ffff80001015078c: orr w0, w0, #0x80000000 0.00 : ffff800010150790: str w0, [x19, #208] : return; 0.00 : ffff800010150794: ldr x20, [x29, #24] 0.00 : ffff800010150798: ldr x22, [x29, #40] 0.00 : ffff80001015079c: b ffff8000101506c4 : report_bad_irq(): : if (count > 0) { 0.00 : ffff8000101507a0: adrp x2, ffff800011921000 0.00 : ffff8000101507a4: add x2, x2, #0xdb0 0.00 : ffff8000101507a8: ldr w3, [x2, #40] 0.00 : ffff8000101507ac: cmp w3, #0x0 0.00 : ffff8000101507b0: b.gt ffff800010150878 0.00 : ffff8000101507b4: ldr x20, [x29, #24] 0.00 : ffff8000101507b8: b ffff8000101506c4 : note_interrupt(): : if (time_after(jiffies, desc->last_unhandled + HZ/10)) 2.09 : ffff8000101507bc: adrp x1, ffff800011907000 9.43 : ffff8000101507c0: ldr x2, [x19, #192] 0.00 : ffff8000101507c4: mov x0, #0x19 // #25 2.54 : ffff8000101507c8: ldr x3, [x1, #2432] 0.00 : ffff8000101507cc: sub x0, x0, x3 0.00 : ffff8000101507d0: cmn x0, x2 0.00 : ffff8000101507d4: b.mi ffff80001015089c // b.first : desc->irqs_unhandled++; 2.65 : ffff8000101507d8: ldr w0, [x19, #200] 0.00 : ffff8000101507dc: add w0, w0, #0x1 6.37 : ffff8000101507e0: str w0, [x19, #200] : try_misrouted_irq(): : if (!irqfixup) 0.00 : ffff8000101507e4: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> : note_interrupt(): : desc->last_unhandled = jiffies; 2.04 : ffff8000101507e8: ldr x1, [x1, #2432] 0.94 : ffff8000101507ec: str x1, [x19, #192] : try_misrouted_irq(): : if (!irqfixup) 0.06 : ffff8000101507f0: ldr w0, [x0, #648] : note_interrupt(): : irq = irq_desc_get_irq(desc); 9.44 : ffff8000101507f4: ldr w22, [x19, #92] : try_misrouted_irq(): : if (!irqfixup) 0.00 : ffff8000101507f8: cbz w0, ffff800010150914 0.00 : ffff8000101507fc: stp x23, x24, [x29, #48] 0.00 : ffff800010150800: mov w20, #0x0 // #0 0.00 : ffff800010150804: mov w24, w22 0.00 : ffff800010150808: str x21, [x29, #32] 0.00 : ffff80001015080c: str x26, [x29, #72] 0.00 : ffff800010150810: b ffff800010150640 : note_interrupt(): : if (unlikely(desc->irqs_unhandled > 99900)) { 0.00 : ffff800010150814: ldr w1, [x19, #200] 0.00 : ffff800010150818: mov w0, #0x863c // #34364 : desc->irq_count = 0; 0.00 : ffff80001015081c: str wzr, [x19, #188] : if (unlikely(desc->irqs_unhandled > 99900)) { 0.00 : ffff800010150820: movk w0, #0x1, lsl #16 0.00 : ffff800010150824: cmp w1, w0 0.00 : ffff800010150828: b.hi ffff8000101508b4 // b.pmore : desc->irqs_unhandled = 0; 0.00 : ffff80001015082c: str wzr, [x19, #200] 0.00 : ffff800010150830: ldr x20, [x29, #24] 0.00 : ffff800010150834: ldr x22, [x29, #40] 0.00 : ffff800010150838: b ffff8000101506c4 : __ll_sc_atomic_add_return(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff80001015083c: adrp x21, ffff800011b20000 <__log_buf+0x1fc10> 0.00 : ffff800010150840: add x1, x21, #0x880 0.00 : ffff800010150844: b ffff800010150950 0.00 : ffff800010150848: b ffff80001015065c : __ll_sc_atomic_sub(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff80001015084c: mov w0, #0x1 // #1 0.00 : ffff800010150850: add x3, x21, #0x880 0.00 : ffff800010150854: b ffff80001015096c 0.00 : ffff800010150858: b ffff800010150680 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 4.35 : ffff80001015085c: ldr w1, [x19, #204] : note_interrupt(): : handled |= SPURIOUS_DEFERRED; 0.03 : ffff800010150860: orr w1, w1, #0x80000000 : if (handled != desc->threads_handled_last) { 0.00 : ffff800010150864: cmp w1, w0 0.00 : ffff800010150868: b.eq ffff8000101507bc // b.none : action_ret = IRQ_HANDLED; 0.42 : ffff80001015086c: mov w20, #0x1 // #1 : desc->threads_handled_last = handled; 1.37 : ffff800010150870: str w1, [x19, #208] 0.00 : ffff800010150874: b ffff800010150604 : report_bad_irq(): : count--; 0.00 : ffff800010150878: sub w3, w3, #0x1 0.00 : ffff80001015087c: str w3, [x2, #40] : __report_bad_irq(desc, action_ret); 0.00 : ffff800010150880: bl ffff8000101502c0 <__report_bad_irq> 0.00 : ffff800010150884: ldr x20, [x29, #24] 0.00 : ffff800010150888: b ffff8000101506c4 0.00 : ffff80001015088c: ldr x21, [x29, #32] 0.00 : ffff800010150890: ldp x23, x24, [x29, #48] 0.00 : ffff800010150894: ldr x26, [x29, #72] 0.00 : ffff800010150898: b ffff8000101506a0 : note_interrupt(): : desc->irqs_unhandled = 1; 0.00 : ffff80001015089c: mov w0, #0x1 // #1 0.00 : ffff8000101508a0: str w0, [x19, #200] 0.00 : ffff8000101508a4: b ffff8000101507e4 0.00 : ffff8000101508a8: ldr x25, [x29, #64] 0.00 : ffff8000101508ac: ldr x27, [x29, #80] 0.00 : ffff8000101508b0: b ffff800010150668 : __report_bad_irq(desc, action_ret); 0.00 : ffff8000101508b4: mov w1, w20 0.00 : ffff8000101508b8: mov x0, x19 0.00 : ffff8000101508bc: bl ffff8000101502c0 <__report_bad_irq> : printk(KERN_EMERG "Disabling IRQ #%d\n", irq); 0.00 : ffff8000101508c0: mov w1, w22 0.00 : ffff8000101508c4: adrp x0, ffff8000111c6000 0.00 : ffff8000101508c8: add x0, x0, #0xa30 0.00 : ffff8000101508cc: bl ffff80001014a85c : desc->depth++; 0.00 : ffff8000101508d0: ldp w2, w1, [x19, #172] : irq_disable(desc); 0.00 : ffff8000101508d4: mov x0, x19 : desc->istate |= IRQS_SPURIOUS_DISABLED; 0.00 : ffff8000101508d8: orr w2, w2, #0x2 : desc->depth++; 0.00 : ffff8000101508dc: add w1, w1, #0x1 0.00 : ffff8000101508e0: stp w2, w1, [x19, #172] : irq_disable(desc); 0.00 : ffff8000101508e4: bl ffff8000101522d8 : mod_timer(&poll_spurious_irq_timer, 0.00 : ffff8000101508e8: adrp x1, ffff800011907000 0.00 : ffff8000101508ec: adrp x0, ffff800011921000 0.00 : ffff8000101508f0: add x0, x0, #0xdb0 0.00 : ffff8000101508f4: ldr x1, [x1, #2432] 0.00 : ffff8000101508f8: add x1, x1, #0x19 0.00 : ffff8000101508fc: bl ffff80001016a270 0.00 : ffff800010150900: b ffff80001015082c 0.00 : ffff800010150904: ldr x25, [x29, #64] : misrouted_irq(): : int i, ok = 0; 0.00 : ffff800010150908: mov w26, #0x0 // #0 0.00 : ffff80001015090c: ldr x27, [x29, #80] 0.00 : ffff800010150910: b ffff800010150668 : try_misrouted_irq(): : if (!irqfixup) 1.03 : ffff800010150914: mov w20, #0x0 // #0 0.00 : ffff800010150918: b ffff8000101506a0 Percent | Source code & Disassembly of vmlinux for cycles (3253 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107cfe70 : : nvme_complete_rq(): : blk_mq_requeue_request(req, false); : blk_mq_delay_kick_requeue_list(req->q, delay); : } : : void nvme_complete_rq(struct request *req) : { 1.42 : ffff8000107cfe70: stp x29, x30, [sp, #-48]! 0.00 : ffff8000107cfe74: mov x29, sp 3.42 : ffff8000107cfe78: stp x19, x20, [sp, #16] : blk_status_t status = nvme_error_status(nvme_req(req)->status); 0.00 : ffff8000107cfe7c: add x20, x0, #0x118 : { 0.50 : ffff8000107cfe80: stp x21, x22, [sp, #32] 0.00 : ffff8000107cfe84: mov x19, x0 : blk_status_t status = nvme_error_status(nvme_req(req)->status); 0.62 : ffff8000107cfe88: ldrh w0, [x20, #18] 0.00 : ffff8000107cfe8c: bl ffff8000107cb878 0.21 : ffff8000107cfe90: and w22, w0, #0xff : nvme_cleanup_cmd(): : return 0; : } : : void nvme_cleanup_cmd(struct request *req) : { : if (req->rq_flags & RQF_SPECIAL_PAYLOAD) { 0.06 : ffff8000107cfe94: ldr w0, [x19, #28] : nvme_complete_rq(): : blk_status_t status = nvme_error_status(nvme_req(req)->status); 0.00 : ffff8000107cfe98: mov w21, w22 : nvme_cleanup_cmd(): : if (req->rq_flags & RQF_SPECIAL_PAYLOAD) { 0.00 : ffff8000107cfe9c: tbnz w0, #18, ffff8000107cfed4 : nvme_complete_rq(): : if (nvme_req(req)->ctrl->kas) 4.98 : ffff8000107cfea0: ldr x0, [x20, #24] 13.02 : ffff8000107cfea4: ldrh w1, [x0, #1256] 0.00 : ffff8000107cfea8: cbz w1, ffff8000107cfeb4 : nvme_req(req)->ctrl->comp_seen = true; 0.00 : ffff8000107cfeac: mov w1, #0x1 // #1 0.00 : ffff8000107cfeb0: strb w1, [x0] : if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) { 56.67 : ffff8000107cfeb4: cbnz w22, ffff8000107cfee0 : blk_mq_end_request(req, status); 8.18 : ffff8000107cfeb8: mov w1, w21 0.00 : ffff8000107cfebc: mov x0, x19 0.00 : ffff8000107cfec0: bl ffff800010465d98 : } 1.05 : ffff8000107cfec4: ldp x19, x20, [sp, #16] 2.33 : ffff8000107cfec8: ldp x21, x22, [sp, #32] 7.54 : ffff8000107cfecc: ldp x29, x30, [sp], #48 0.00 : ffff8000107cfed0: ret : nvme_cleanup_cmd(): 0.00 : ffff8000107cfed4: mov x0, x19 0.00 : ffff8000107cfed8: bl ffff8000107cfdd0 0.00 : ffff8000107cfedc: b ffff8000107cfea0 : nvme_req_needs_retry(): : if (blk_noretry_request(req)) 0.00 : ffff8000107cfee0: ldr w0, [x19, #24] 0.00 : ffff8000107cfee4: tst w0, #0x700 0.00 : ffff8000107cfee8: b.ne ffff8000107cfeb8 // b.any : if (nvme_req(req)->status & NVME_SC_DNR) 0.00 : ffff8000107cfeec: ldrh w1, [x20, #18] 0.00 : ffff8000107cfef0: tbnz w1, #14, ffff8000107cfeb8 : if (nvme_req(req)->retries >= nvme_max_retries) 0.00 : ffff8000107cfef4: adrp x3, ffff800011a93000 0.00 : ffff8000107cfef8: ldrb w2, [x20, #16] 0.00 : ffff8000107cfefc: ldrb w3, [x3, #832] 0.00 : ffff8000107cff00: cmp w3, w2 0.00 : ffff8000107cff04: b.ls ffff8000107cfeb8 // b.plast : nvme_complete_rq(): : if ((req->cmd_flags & REQ_NVME_MPATH) && 0.00 : ffff8000107cff08: tbz w0, #26, ffff8000107cff20 : blk_path_error(): : * %false - retrying failover path will not help : * %true - may succeed if retried : */ : static inline bool blk_path_error(blk_status_t error) : { : switch (error) { 0.00 : ffff8000107cff0c: cmp w22, #0x3 0.00 : ffff8000107cff10: b.eq ffff8000107cff20 // b.none 0.00 : ffff8000107cff14: b.hi ffff8000107cff88 // b.pmore 0.00 : ffff8000107cff18: cmp w22, #0x1 0.00 : ffff8000107cff1c: b.ne ffff8000107cfec4 // b.any : nvme_complete_rq(): : if (!blk_queue_dying(req->q)) { 0.00 : ffff8000107cff20: ldr x0, [x19] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000107cff24: ldr x3, [x0, #104] : nvme_complete_rq(): 0.00 : ffff8000107cff28: tst w3, #0x2 0.00 : ffff8000107cff2c: b.ne ffff8000107cfeb8 // b.any : nvme_retry_req(): : struct nvme_ns *ns = req->q->queuedata; 0.00 : ffff8000107cff30: ldr x0, [x0, #96] : crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11; 0.00 : ffff8000107cff34: ubfx x1, x1, #11, #2 : unsigned long delay = 0; 0.00 : ffff8000107cff38: mov x21, #0x0 // #0 : if (ns && crd) 0.00 : ffff8000107cff3c: cmp x0, #0x0 0.00 : ffff8000107cff40: ccmp w1, #0x0, #0x4, ne // ne = any 0.00 : ffff8000107cff44: b.eq ffff8000107cff64 // b.none : delay = ns->ctrl->crdt[crd - 1] * 100; 0.00 : ffff8000107cff48: ldr x0, [x0, #16] 0.00 : ffff8000107cff4c: sub w1, w1, #0x1 0.00 : ffff8000107cff50: mov w3, #0x64 // #100 0.00 : ffff8000107cff54: add x0, x0, w1, sxtw #1 0.00 : ffff8000107cff58: ldrh w21, [x0, #1220] 0.00 : ffff8000107cff5c: mul w21, w21, w3 0.00 : ffff8000107cff60: sxtw x21, w21 : nvme_req(req)->retries++; 0.00 : ffff8000107cff64: add w2, w2, #0x1 0.00 : ffff8000107cff68: strb w2, [x20, #16] : blk_mq_requeue_request(req, false); 0.00 : ffff8000107cff6c: mov x0, x19 0.00 : ffff8000107cff70: mov w1, #0x0 // #0 0.00 : ffff8000107cff74: bl ffff800010466a68 : blk_mq_delay_kick_requeue_list(req->q, delay); 0.00 : ffff8000107cff78: ldr x0, [x19] 0.00 : ffff8000107cff7c: mov x1, x21 0.00 : ffff8000107cff80: bl ffff800010464538 0.00 : ffff8000107cff84: b ffff8000107cfec4 : blk_path_error(): 0.00 : ffff8000107cff88: sub w22, w22, #0x5 0.00 : ffff8000107cff8c: cmp w22, #0x3 0.00 : ffff8000107cff90: b.hi ffff8000107cfec4 // b.pmore 0.00 : ffff8000107cff94: b ffff8000107cff20 Percent | Source code & Disassembly of vmlinux for cycles (3158 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011c540 : : update_load_add(): : unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; : #endif : : static inline void update_load_add(struct load_weight *lw, unsigned long inc) : { : lw->weight += inc; 5.93 : ffff80001011c540: ldr x3, [x1] 6.15 : ffff80001011c544: ldr x2, [x0] : lw->inv_weight = 0; 4.15 : ffff80001011c548: str wzr, [x0, #8] : lw->weight += inc; 0.00 : ffff80001011c54c: add x2, x2, x3 5.70 : ffff80001011c550: str x2, [x0] : account_entity_enqueue(): : static void : account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) : { : update_load_add(&cfs_rq->load, se->load.weight); : #ifdef CONFIG_SMP : if (entity_is_task(se)) { 5.67 : ffff80001011c554: ldr x2, [x1, #136] 0.16 : ffff80001011c558: cbnz x2, ffff80001011c5b8 : struct rq *rq = rq_of(cfs_rq); 3.82 : ffff80001011c55c: ldr x2, [x0, #304] : cpu_to_node(): : #endif : : #ifndef cpu_to_node : static inline int cpu_to_node(int cpu) : { : return per_cpu(numa_node, cpu); 0.06 : ffff80001011c560: adrp x5, ffff800011909000 : account_numa_enqueue(): : rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 0.79 : ffff80001011c564: ldr w4, [x1, #2000] : cpu_to_node(): 0.00 : ffff80001011c568: add x5, x5, #0x928 2.28 : ffff80001011c56c: adrp x3, ffff800011526000 0.00 : ffff80001011c570: add x3, x3, #0xd58 : account_numa_enqueue(): 2.63 : ffff80001011c574: cmn w4, #0x1 : list_add(): : * Insert a new entry after the specified head. : * This is good for implementing stacks. : */ : static inline void list_add(struct list_head *new, struct list_head *head) : { : __list_add(new, head, head->next); 0.41 : ffff80001011c578: ldr x7, [x2, #2576] : account_numa_enqueue(): : rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 3.07 : ffff80001011c57c: ldp w6, w4, [x2, #8] : account_entity_enqueue(): : : account_numa_enqueue(rq, task_of(se)); : list_add(&se->group_node, &rq->cfs_tasks); 6.42 : ffff80001011c580: add x8, x2, #0xa10 : account_numa_enqueue(): : rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 0.00 : ffff80001011c584: cinc w6, w6, ne // ne = any 8.18 : ffff80001011c588: str w6, [x2, #8] : account_entity_enqueue(): : list_add(&se->group_node, &rq->cfs_tasks); 0.00 : ffff80001011c58c: add x6, x1, #0x30 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.05 : ffff80001011c590: ldur w10, [x1, #-124] : account_numa_enqueue(): : rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 7.52 : ffff80001011c594: ldr w9, [x1, #2000] : cpu_to_node(): 1.96 : ffff80001011c598: ldr x5, [x5, w10, sxtw #3] : account_numa_enqueue(): 5.73 : ffff80001011c59c: ldr w3, [x5, x3] 0.43 : ffff80001011c5a0: cmp w3, w9 0.00 : ffff80001011c5a4: cinc w3, w4, eq // eq = none 16.72 : ffff80001011c5a8: str w3, [x2, #12] : __list_add(): : next->prev = new; 1.05 : ffff80001011c5ac: str x6, [x7, #8] : new->prev = prev; 1.09 : ffff80001011c5b0: stp x7, x8, [x1, #48] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 1.65 : ffff80001011c5b4: str x6, [x2, #2576] : account_entity_enqueue(): : } : #endif : cfs_rq->nr_running++; 4.41 : ffff80001011c5b8: ldr w1, [x0, #24] 0.00 : ffff80001011c5bc: add w1, w1, #0x1 1.58 : ffff80001011c5c0: str w1, [x0, #24] : } 0.41 : ffff80001011c5c4: ret Percent | Source code & Disassembly of vmlinux for cycles (3133 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107017a0 <_mix_pool_bytes>: : _mix_pool_bytes(): : static void _mix_pool_bytes(struct entropy_store *r, const void *in, : int nbytes) : { : unsigned long i, tap1, tap2, tap3, tap4, tap5; : int input_rotate; : int wordmask = r->poolinfo->poolwords - 1; 0.00 : ffff8000107017a0: ldr x4, [x0] : tap2 = r->poolinfo->tap2; : tap3 = r->poolinfo->tap3; : tap4 = r->poolinfo->tap4; : tap5 = r->poolinfo->tap5; : : input_rotate = r->input_rotate; 2.62 : ffff8000107017a4: ldrh w7, [x0, #30] : i = r->add_ptr; 0.06 : ffff8000107017a8: ldrh w3, [x0, #28] : tap2 = r->poolinfo->tap2; 0.39 : ffff8000107017ac: ldpsw x12, x13, [x4, #16] : tap4 = r->poolinfo->tap4; 3.71 : ffff8000107017b0: ldpsw x14, x15, [x4, #24] : tap5 = r->poolinfo->tap5; 0.19 : ffff8000107017b4: ldrsw x16, [x4, #32] : int wordmask = r->poolinfo->poolwords - 1; 0.06 : ffff8000107017b8: ldr w6, [x4, #4] : : /* mix one byte at a time to simplify size handling and churn faster */ : while (nbytes--) { 0.00 : ffff8000107017bc: cbz w2, ffff8000107018b0 <_mix_pool_bytes+0x110> : { 0.03 : ffff8000107017c0: stp x29, x30, [sp, #-48]! 0.00 : ffff8000107017c4: sub w6, w6, #0x1 0.00 : ffff8000107017c8: sub w10, w2, #0x1 0.00 : ffff8000107017cc: adrp x11, ffff800010ed7000 0.00 : ffff8000107017d0: add x10, x10, #0x1 0.00 : ffff8000107017d4: mov x29, sp 0.00 : ffff8000107017d8: sxtw x6, w6 0.00 : ffff8000107017dc: add x10, x1, x10 : w ^= r->pool[(i + tap3) & wordmask]; : w ^= r->pool[(i + tap4) & wordmask]; : w ^= r->pool[(i + tap5) & wordmask]; : : /* Mix the result back in with a twist */ : r->pool[i] = (w >> 3) ^ twist_table[w & 7]; 0.00 : ffff8000107017e0: add x11, x11, #0x5e8 : { 0.13 : ffff8000107017e4: stp x19, x20, [sp, #16] : * Normally, we add 7 bits of rotation to the pool. : * At the beginning of the pool, add an extra 7 bits : * rotation, so that successive passes spread the : * input bits across the pool evenly. : */ : input_rotate = (input_rotate + (i ? 7 : 14)) & 31; 0.00 : ffff8000107017e8: mov w18, #0xe // #14 0.00 : ffff8000107017ec: mov w17, #0x7 // #7 : { 0.00 : ffff8000107017f0: str x21, [sp, #32] 0.00 : ffff8000107017f4: nop : i = (i - 1) & wordmask; 0.00 : ffff8000107017f8: sub x3, x3, #0x1 : w ^= r->pool[i]; 0.00 : ffff8000107017fc: ldr x4, [x0, #8] : i = (i - 1) & wordmask; 0.00 : ffff800010701800: and x3, x3, x6 : w = rol32(*bytes++, input_rotate); 0.13 : ffff800010701804: ldrb w5, [x1], #1 : w ^= r->pool[(i + tap3) & wordmask]; 0.06 : ffff800010701808: add x21, x14, x3 : w ^= r->pool[(i + tap4) & wordmask]; 0.00 : ffff80001070180c: add x20, x15, x3 : w ^= r->pool[(i + tap1) & wordmask]; 0.29 : ffff800010701810: add x9, x12, x3 : w ^= r->pool[(i + tap2) & wordmask]; 1.59 : ffff800010701814: add x19, x13, x3 : w ^= r->pool[(i + tap3) & wordmask]; 0.00 : ffff800010701818: and x21, x21, x6 : w ^= r->pool[(i + tap4) & wordmask]; 0.00 : ffff80001070181c: and x20, x20, x6 : w ^= r->pool[(i + tap1) & wordmask]; 0.00 : ffff800010701820: and x9, x9, x6 : w ^= r->pool[(i + tap2) & wordmask]; 0.00 : ffff800010701824: and x19, x19, x6 : w ^= r->pool[(i + tap5) & wordmask]; 0.00 : ffff800010701828: add x30, x16, x3 : input_rotate = (input_rotate + (i ? 7 : 14)) & 31; 0.00 : ffff80001070182c: cmp x3, #0x0 : w ^= r->pool[(i + tap5) & wordmask]; 0.10 : ffff800010701830: ldr w2, [x4, x21, lsl #2] 0.03 : ffff800010701834: and x30, x30, x6 35.88 : ffff800010701838: ldr w21, [x4, x20, lsl #2] : rol32(): : * @word: value to rotate : * @shift: bits to roll : */ : static inline __u32 rol32(__u32 word, unsigned int shift) : { : return (word << (shift & 31)) | (word >> ((-shift) & 31)); 0.06 : ffff80001070183c: neg w20, w7 : _mix_pool_bytes(): 12.32 : ffff800010701840: ldr w19, [x4, x19, lsl #2] 3.02 : ffff800010701844: ldr w9, [x4, x9, lsl #2] 0.00 : ffff800010701848: eor w2, w2, w21 1.36 : ffff80001070184c: ldr w8, [x4, x30, lsl #2] 0.07 : ffff800010701850: eor w9, w9, w19 : rol32(): 0.32 : ffff800010701854: lsl w19, w5, w7 : _mix_pool_bytes(): 1.14 : ffff800010701858: eor w2, w2, w9 13.49 : ffff80001070185c: ldr w9, [x4, x3, lsl #2] : rol32(): 0.00 : ffff800010701860: lsr w5, w5, w20 0.00 : ffff800010701864: orr w5, w5, w19 : _mix_pool_bytes(): 0.00 : ffff800010701868: eor w8, w8, w9 0.87 : ffff80001070186c: eor w2, w2, w8 0.00 : ffff800010701870: eor w2, w2, w5 : input_rotate = (input_rotate + (i ? 7 : 14)) & 31; 0.00 : ffff800010701874: csel w5, w18, w17, eq // eq = none : r->pool[i] = (w >> 3) ^ twist_table[w & 7]; 0.00 : ffff800010701878: and x8, x2, #0x7 : input_rotate = (input_rotate + (i ? 7 : 14)) & 31; 0.07 : ffff80001070187c: add w7, w5, w7 0.00 : ffff800010701880: and w7, w7, #0x1f : while (nbytes--) { 0.10 : ffff800010701884: cmp x1, x10 : r->pool[i] = (w >> 3) ^ twist_table[w & 7]; 0.22 : ffff800010701888: ldr w5, [x11, x8, lsl #2] 0.51 : ffff80001070188c: eor w2, w5, w2, lsr #3 20.96 : ffff800010701890: str w2, [x4, x3, lsl #2] : while (nbytes--) { 0.00 : ffff800010701894: b.ne ffff8000107017f8 <_mix_pool_bytes+0x58> // b.any : } : : r->input_rotate = input_rotate; : r->add_ptr = i; : } 0.00 : ffff800010701898: ldp x19, x20, [sp, #16] : r->add_ptr = i; 0.00 : ffff80001070189c: strh w3, [x0, #28] : } 0.00 : ffff8000107018a0: ldr x21, [sp, #32] : r->input_rotate = input_rotate; 0.22 : ffff8000107018a4: strh w7, [x0, #30] : } 0.00 : ffff8000107018a8: ldp x29, x30, [sp], #48 0.00 : ffff8000107018ac: ret : r->add_ptr = i; 0.00 : ffff8000107018b0: strh w3, [x0, #28] : r->input_rotate = input_rotate; 0.00 : ffff8000107018b4: strh w7, [x0, #30] 0.00 : ffff8000107018b8: ret Percent | Source code & Disassembly of vmlinux for cycles (2968 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d8f58 : : refill_reqs_available(): : unsigned tail) : { : unsigned events_in_ring, completed; : : /* Clamp head since userland can write to it. */ : head %= ctx->nr_events; 1.66 : ffff8000102d8f58: ldr w5, [x0, #144] : if (head <= tail) : events_in_ring = tail - head; : else : events_in_ring = ctx->nr_events - (head - tail); : : completed = ctx->completed_events; 33.25 : ffff8000102d8f5c: ldr w7, [x0, #452] : events_in_ring = tail - head; 0.03 : ffff8000102d8f60: add w3, w5, w2 : head %= ctx->nr_events; 0.00 : ffff8000102d8f64: udiv w4, w1, w5 0.00 : ffff8000102d8f68: msub w1, w4, w5, w1 : events_in_ring = tail - head; 24.94 : ffff8000102d8f6c: cmp w1, w2 0.00 : ffff8000102d8f70: sub w2, w2, w1 0.00 : ffff8000102d8f74: sub w1, w3, w1 0.00 : ffff8000102d8f78: csel w1, w1, w2, hi // hi = pmore : if (events_in_ring < completed) 0.27 : ffff8000102d8f7c: cmp w1, w7 0.00 : ffff8000102d8f80: b.cc ffff8000102d8f88 // b.lo, b.ul, b.last 0.03 : ffff8000102d8f84: ret : { 3.27 : ffff8000102d8f88: stp x29, x30, [sp, #-16]! 0.00 : ffff8000102d8f8c: mov x29, sp : completed = 0; : : if (!completed) : return; : : ctx->completed_events -= completed; 36.50 : ffff8000102d8f90: str w1, [x0, #452] : put_reqs_available(ctx, completed); 0.00 : ffff8000102d8f94: sub w1, w7, w1 0.00 : ffff8000102d8f98: bl ffff8000102d8ec8 : } 0.00 : ffff8000102d8f9c: ldp x29, x30, [sp], #16 0.03 : ffff8000102d8fa0: ret Percent | Source code & Disassembly of vmlinux for cycles (2912 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010134868 <__update_load_avg_se>: : ___update_load_sum(): : ___update_load_sum(u64 now, struct sched_avg *sa, : unsigned long load, unsigned long runnable, int running) : { : u64 delta; : : delta = now - sa->last_update_time; 0.45 : ffff800010134868: ldr x4, [x2, #192] : /* : * This should only happen when time goes backwards, which it : * unfortunately does during sched clock init when we swap over to TSC. : */ : if ((s64)delta < 0) { 0.00 : ffff80001013486c: subs x3, x0, x4 0.00 : ffff800010134870: b.mi ffff8000101349ac <__update_load_avg_se+0x144> // b.first : : /* : * Use 1024ns as the unit of measurement since it's a reasonable : * approximation of 1us and fast to compute. : */ : delta >>= 10; 3.52 : ffff800010134874: lsr x5, x3, #10 : if (!delta) 0.00 : ffff800010134878: cbz x5, ffff8000101349a4 <__update_load_avg_se+0x13c> : __update_load_avg_se(): : : return 0; : } : : int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se) : { 12.82 : ffff80001013487c: stp x29, x30, [sp, #-16]! 0.24 : ffff800010134880: mov x7, x2 0.00 : ffff800010134884: add x8, x2, #0xc0 : ___update_load_sum(): : sa->last_update_time += delta << 10; 0.00 : ffff800010134888: and x3, x3, #0xfffffffffffffc00 : __update_load_avg_se(): : { 21.20 : ffff80001013488c: mov x29, sp : ___update_load_sum(): : sa->last_update_time += delta << 10; 0.00 : ffff800010134890: add x3, x3, x4 : __update_load_avg_se(): : if (___update_load_sum(now, &se->avg, !!se->on_rq, !!se->on_rq, 0.00 : ffff800010134894: ldr w6, [x2, #64] : cfs_rq->curr == se)) { 9.72 : ffff800010134898: ldr x2, [x1, #72] 1.00 : ffff80001013489c: mov w0, w5 : ___update_load_sum(): : sa->last_update_time += delta << 10; 5.42 : ffff8000101348a0: str x3, [x7, #192] : __update_load_avg_se(): : if (___update_load_sum(now, &se->avg, !!se->on_rq, !!se->on_rq, 0.00 : ffff8000101348a4: cmp w6, #0x0 0.00 : ffff8000101348a8: cset x11, ne // ne = any 10.38 : ffff8000101348ac: ldr w1, [x8, #28] 0.00 : ffff8000101348b0: add x5, x5, w1, uxtw 0.00 : ffff8000101348b4: lsr x9, x5, #10 : ___update_load_sum(): : if (!load) 0.00 : ffff8000101348b8: cbnz w6, ffff8000101349b8 <__update_load_avg_se+0x150> : runnable = running = 0; 0.24 : ffff8000101348bc: mov w12, #0x0 // #0 0.00 : ffff8000101348c0: mov x13, #0x0 // #0 : accumulate_sum(): : if (periods) { 0.00 : ffff8000101348c4: cbz x9, ffff8000101349e4 <__update_load_avg_se+0x17c> : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.28 : ffff8000101348c8: cmp x9, #0x7e0 : accumulate_sum(): : sa->load_sum = decay_load(sa->load_sum, periods); 0.17 : ffff8000101348cc: ldr x3, [x8, #8] : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.00 : ffff8000101348d0: b.hi ffff800010134a18 <__update_load_avg_se+0x1b0> // b.pmore : if (unlikely(local_n >= LOAD_AVG_PERIOD)) { 0.49 : ffff8000101348d4: cmp x9, #0x1f 0.03 : ffff8000101348d8: ldr w14, [x8, #24] 0.03 : ffff8000101348dc: ldr x4, [x8, #16] 0.00 : ffff8000101348e0: b.hi ffff800010134a24 <__update_load_avg_se+0x1bc> // b.pmore : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.04 : ffff8000101348e4: adrp x10, ffff800010d06000 0.00 : ffff8000101348e8: add x10, x10, #0x450 : mul_u64_u32_shr(): : #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) : : #ifndef mul_u64_u32_shr : static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) : { : return (u64)(((unsigned __int128)a * mul) >> shift); 0.42 : ffff8000101348ec: ldr w2, [x10, w9, uxtw #2] 0.00 : ffff8000101348f0: mul x15, x2, x3 0.00 : ffff8000101348f4: mul x10, x2, x4 0.24 : ffff8000101348f8: umulh x3, x2, x3 0.07 : ffff8000101348fc: umulh x4, x2, x4 0.00 : ffff800010134900: extr x3, x3, x15, #32 0.00 : ffff800010134904: extr x4, x4, x10, #32 0.00 : ffff800010134908: stp x3, x4, [x8, #8] 0.00 : ffff80001013490c: mul x2, x14, x2 0.00 : ffff800010134910: lsr x2, x2, #32 : accumulate_sum(): : sa->util_sum = decay_load((u64)(sa->util_sum), periods); 0.03 : ffff800010134914: str w2, [x8, #24] : delta %= 1024; 0.00 : ffff800010134918: and x14, x5, #0x3ff 0.00 : ffff80001013491c: mov w10, w14 : if (load) { 0.00 : ffff800010134920: cbnz w6, ffff8000101349fc <__update_load_avg_se+0x194> : sa->period_contrib = delta; 0.17 : ffff800010134924: str w14, [x8, #28] : if (runnable) 0.21 : ffff800010134928: cbz x13, ffff800010134938 <__update_load_avg_se+0xd0> : sa->runnable_load_sum += runnable * contrib; 0.10 : ffff80001013492c: ldr x1, [x8, #16] 0.00 : ffff800010134930: add x1, x1, w0, uxtw 0.14 : ffff800010134934: str x1, [x8, #16] : if (running) 0.00 : ffff800010134938: cbz w12, ffff800010134948 <__update_load_avg_se+0xe0> : sa->util_sum += contrib << SCHED_CAPACITY_SHIFT; 0.00 : ffff80001013493c: ldr w1, [x8, #24] 0.00 : ffff800010134940: add w0, w1, w0, lsl #10 0.00 : ffff800010134944: str w0, [x8, #24] : ___update_load_sum(): : if (!accumulate_sum(delta, sa, load, runnable, running)) 0.00 : ffff800010134948: cbz w9, ffff8000101349e8 <__update_load_avg_se+0x180> : se_weight(): : /* : * XXX we want to get rid of these helpers and use the full load resolution. : */ : static inline long se_weight(struct sched_entity *se) : { : return scale_load_down(se->load.weight); 0.03 : ffff80001013494c: ldr x2, [x7] : ___update_load_avg(): : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.03 : ffff800010134950: mov w0, #0xb67e // #46718 : se_runnable(): : } : : static inline long se_runnable(struct sched_entity *se) : { : return scale_load_down(se->runnable_weight); 2.51 : ffff800010134954: ldr x1, [x7, #16] : ___update_load_avg(): 0.00 : ffff800010134958: add w5, w10, w0 : sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider); 0.10 : ffff80001013495c: ldp x9, x6, [x8, #8] : se_weight(): : return scale_load_down(se->load.weight); 0.14 : ffff800010134960: lsr x2, x2, #10 : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff800010134964: lsr x1, x1, #10 : ___update_load_avg(): : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 0.31 : ffff800010134968: ldr w4, [x8, #24] : __update_load_avg_se(): : : ___update_load_avg(&se->avg, se_weight(se), se_runnable(se)); : cfs_se_util_change(&se->avg); : trace_pelt_se_tp(se); : return 1; 0.00 : ffff80001013496c: mov w0, #0x1 // #1 : ___update_load_avg(): : sa->load_avg = div_u64(load * sa->load_sum, divider); 0.03 : ffff800010134970: mul x2, x2, x9 : sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider); 0.00 : ffff800010134974: mul x1, x1, x6 : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 0.10 : ffff800010134978: udiv w3, w4, w5 : div_u64_rem(): : return dividend / divisor; 0.00 : ffff80001013497c: udiv x2, x2, x5 0.00 : ffff800010134980: udiv x1, x1, x5 : ___update_load_avg(): : sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider); 0.10 : ffff800010134984: stp x2, x1, [x8, #32] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010134988: str x3, [x7, #240] : cfs_se_util_change(): : : if (!sched_feat(UTIL_EST)) : return; : : /* Avoid store if the flag has been already set */ : enqueued = avg->util_est.enqueued; 0.07 : ffff80001013498c: ldr w1, [x8, #56] : if (!(enqueued & UTIL_AVG_UNCHANGED)) 0.00 : ffff800010134990: tbz w1, #0, ffff80001013499c <__update_load_avg_se+0x134> : return; : : /* Reset flag to report util_avg has been updated */ : enqueued &= ~UTIL_AVG_UNCHANGED; 0.00 : ffff800010134994: and w1, w1, #0xfffffffe : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010134998: str w1, [x7, #248] : __update_load_avg_se(): : } : : return 0; : } 0.00 : ffff80001013499c: ldp x29, x30, [sp], #16 0.00 : ffff8000101349a0: ret : return 0; 1.12 : ffff8000101349a4: mov w0, #0x0 // #0 : } 0.00 : ffff8000101349a8: ret : ___update_load_sum(): : sa->last_update_time = now; 0.00 : ffff8000101349ac: str x0, [x2, #192] : __update_load_avg_se(): : return 0; 0.00 : ffff8000101349b0: mov w0, #0x0 // #0 0.00 : ffff8000101349b4: ret : if (___update_load_sum(now, &se->avg, !!se->on_rq, !!se->on_rq, 0.00 : ffff8000101349b8: cmp x2, x7 0.00 : ffff8000101349bc: cset w12, eq // eq = none : accumulate_sum(): : if (periods) { 0.04 : ffff8000101349c0: cbnz x9, ffff8000101349f4 <__update_load_avg_se+0x18c> : sa->period_contrib = delta; 0.03 : ffff8000101349c4: mov w10, w5 : __update_load_avg_se(): : if (___update_load_sum(now, &se->avg, !!se->on_rq, !!se->on_rq, 0.00 : ffff8000101349c8: mov x13, x11 : accumulate_sum(): : sa->period_contrib = delta; 0.07 : ffff8000101349cc: str w5, [x8, #28] : sa->load_sum += load * contrib; 0.07 : ffff8000101349d0: ldr x2, [x8, #8] 0.00 : ffff8000101349d4: mov w1, w0 0.00 : ffff8000101349d8: madd x11, x1, x11, x2 0.07 : ffff8000101349dc: str x11, [x8, #8] 0.00 : ffff8000101349e0: b ffff800010134928 <__update_load_avg_se+0xc0> : sa->period_contrib = delta; 4.25 : ffff8000101349e4: str w5, [x8, #28] : __update_load_avg_se(): : return 0; 0.20 : ffff8000101349e8: mov w0, #0x0 // #0 : } 23.29 : ffff8000101349ec: ldp x29, x30, [sp], #16 0.00 : ffff8000101349f0: ret : if (___update_load_sum(now, &se->avg, !!se->on_rq, !!se->on_rq, 0.00 : ffff8000101349f4: mov x13, x11 0.00 : ffff8000101349f8: b ffff8000101348c8 <__update_load_avg_se+0x60> : accumulate_sum(): : contrib = __accumulate_pelt_segments(periods, 0.00 : ffff8000101349fc: mov w0, #0x400 // #1024 0.00 : ffff800010134a00: mov w2, w14 0.00 : ffff800010134a04: sub w1, w0, w1 0.00 : ffff800010134a08: mov x0, x9 0.00 : ffff800010134a0c: bl ffff8000101346b8 <__accumulate_pelt_segments> : sa->period_contrib = delta; 0.00 : ffff800010134a10: str w14, [x8, #28] 0.00 : ffff800010134a14: b ffff8000101349d0 <__update_load_avg_se+0x168> : sa->runnable_load_sum = 0.00 : ffff800010134a18: mov w2, #0x0 // #0 0.00 : ffff800010134a1c: stp xzr, xzr, [x8, #8] 0.00 : ffff800010134a20: b ffff800010134914 <__update_load_avg_se+0xac> : decay_load(): : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010134a24: and x15, x9, #0x1f 0.00 : ffff800010134a28: adrp x2, ffff800010d06000 0.00 : ffff800010134a2c: add x2, x2, #0x450 : val >>= local_n / LOAD_AVG_PERIOD; 0.00 : ffff800010134a30: lsr w10, w9, #5 0.00 : ffff800010134a34: lsr x3, x3, x10 : mul_u64_u32_shr(): : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff800010134a38: ldr w2, [x2, x15, lsl #2] : decay_load(): 0.00 : ffff800010134a3c: lsr x4, x4, x10 0.00 : ffff800010134a40: lsr x14, x14, x10 : mul_u64_u32_shr(): 0.00 : ffff800010134a44: mul x15, x3, x2 0.00 : ffff800010134a48: umulh x3, x3, x2 0.00 : ffff800010134a4c: mul x10, x4, x2 0.00 : ffff800010134a50: umulh x4, x4, x2 0.00 : ffff800010134a54: extr x3, x3, x15, #32 0.00 : ffff800010134a58: str x3, [x8, #8] 0.00 : ffff800010134a5c: extr x3, x4, x10, #32 0.00 : ffff800010134a60: str x3, [x8, #16] 0.00 : ffff800010134a64: b ffff80001013490c <__update_load_avg_se+0xa4> Percent | Source code & Disassembly of vmlinux for cycles (2817 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010118870 : : idle_cpu(): : * : * Return: 1 if the CPU is currently idle. 0 otherwise. : */ : int idle_cpu(int cpu) : { : struct rq *rq = cpu_rq(cpu); 1.36 : ffff800010118870: adrp x2, ffff800011909000 0.11 : ffff800010118874: add x2, x2, #0x928 0.39 : ffff800010118878: adrp x1, ffff800011528000 0.18 : ffff80001011887c: add x1, x1, #0xe80 5.45 : ffff800010118880: ldr x0, [x2, w0, sxtw #3] 0.00 : ffff800010118884: add x1, x1, x0 : : if (rq->curr != rq->idle) 4.55 : ffff800010118888: ldr x2, [x1, #2352] 40.99 : ffff80001011888c: ldr x0, [x1, #2360] 0.18 : ffff800010118890: cmp x2, x0 0.00 : ffff800010118894: b.eq ffff8000101188a0 // b.none : available_idle_cpu(): : * Return: 1 if the CPU is currently idle. 0 otherwise. : */ : int available_idle_cpu(int cpu) : { : if (!idle_cpu(cpu)) : return 0; 0.03 : ffff800010118898: mov w0, #0x0 // #0 : : if (vcpu_is_preempted(cpu)) : return 0; : : return 1; : } 0.00 : ffff80001011889c: ret : idle_cpu(): : if (rq->nr_running) 2.26 : ffff8000101188a0: ldr w2, [x1, #4] : available_idle_cpu(): : return 0; 0.00 : ffff8000101188a4: mov w0, #0x0 // #0 : idle_cpu(): : if (rq->nr_running) 0.21 : ffff8000101188a8: cbnz w2, ffff80001011889c : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 44.29 : ffff8000101188ac: ldr x0, [x1, #2976] : idle_cpu(): : if (!llist_empty(&rq->wake_list)) 0.00 : ffff8000101188b0: cmp x0, #0x0 0.00 : ffff8000101188b4: cset w0, eq // eq = none : available_idle_cpu(): : } 0.00 : ffff8000101188b8: ret Percent | Source code & Disassembly of vmlinux for cycles (2780 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010465d98 : : blk_mq_end_request(): : } : } : EXPORT_SYMBOL(__blk_mq_end_request); : : void blk_mq_end_request(struct request *rq, blk_status_t error) : { 1.07 : ffff800010465d98: stp x29, x30, [sp, #-48]! 0.00 : ffff800010465d9c: mov x29, sp 0.97 : ffff800010465da0: stp x19, x20, [sp, #16] 0.00 : ffff800010465da4: and w20, w1, #0xff 0.00 : ffff800010465da8: mov x19, x0 : if (blk_update_request(rq, error, blk_rq_bytes(rq))) 0.00 : ffff800010465dac: mov w1, w20 0.14 : ffff800010465db0: ldr w2, [x0, #40] 0.00 : ffff800010465db4: bl ffff80001045bd28 2.28 : ffff800010465db8: str x21, [x29, #32] 0.00 : ffff800010465dbc: tst w0, #0xff 0.03 : ffff800010465dc0: b.ne ffff800010465ecc // b.any : __blk_mq_end_request(): : if (blk_mq_need_time_stamp(rq)) 1.53 : ffff800010465dc4: ldr w1, [x19, #28] : blk_mq_need_time_stamp(): : return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator; 0.00 : ffff800010465dc8: and w0, w1, #0x3e000 0.00 : ffff800010465dcc: and w0, w0, #0xfffe3fff 0.04 : ffff800010465dd0: cbz w0, ffff800010465e5c : ktime_get_ns(): : return ktime_mono_to_any(mono, TK_OFFS_REAL); : } : : static inline u64 ktime_get_ns(void) : { : return ktime_to_ns(ktime_get()); 34.07 : ffff800010465dd4: bl ffff80001016c938 0.07 : ffff800010465dd8: mov x21, x0 0.03 : ffff800010465ddc: ldr w1, [x19, #28] : __blk_mq_end_request(): : if (rq->rq_flags & RQF_STATS) { 0.28 : ffff800010465de0: tbnz w1, #17, ffff800010465e70 : if (rq->internal_tag != -1) 7.27 : ffff800010465de4: ldr w0, [x19, #36] 0.00 : ffff800010465de8: cmn w0, #0x1 0.00 : ffff800010465dec: b.eq ffff800010465e14 // b.none : blk_mq_sched_completed_request(): : return true; : } : : static inline void blk_mq_sched_completed_request(struct request *rq, u64 now) : { : struct elevator_queue *e = rq->q->elevator; 0.00 : ffff800010465df0: ldr x0, [x19] 0.00 : ffff800010465df4: ldr x0, [x0, #8] : : if (e && e->type->ops.completed_request) 0.00 : ffff800010465df8: cbz x0, ffff800010465e14 0.00 : ffff800010465dfc: ldr x0, [x0] 0.00 : ffff800010465e00: ldr x2, [x0, #136] 0.00 : ffff800010465e04: cbz x2, ffff800010465e14 : e->type->ops.completed_request(rq, now); 0.00 : ffff800010465e08: mov x1, x21 0.00 : ffff800010465e0c: mov x0, x19 0.00 : ffff800010465e10: blr x2 : __blk_mq_end_request(): : blk_account_io_done(rq, now); 3.70 : ffff800010465e14: mov x1, x21 0.00 : ffff800010465e18: mov x0, x19 0.00 : ffff800010465e1c: bl ffff80001045c0c8 : if (rq->end_io) { 0.51 : ffff800010465e20: ldr x2, [x19, #264] 0.36 : ffff800010465e24: cbz x2, ffff800010465e9c : rq_qos_done(rq->q, rq); 0.00 : ffff800010465e28: ldr x0, [x19] 0.00 : ffff800010465e2c: ldr x0, [x0, #24] : rq_qos_done(): : __rq_qos_cleanup(q->rq_qos, bio); : } : : static inline void rq_qos_done(struct request_queue *q, struct request *rq) : { : if (q->rq_qos) 0.00 : ffff800010465e30: cbz x0, ffff800010465e40 : __rq_qos_done(q->rq_qos, rq); 0.00 : ffff800010465e34: mov x1, x19 0.00 : ffff800010465e38: bl ffff800010476170 <__rq_qos_done> 0.00 : ffff800010465e3c: ldr x2, [x19, #264] : __blk_mq_end_request(): : rq->end_io(rq, error); 0.00 : ffff800010465e40: mov w1, w20 0.00 : ffff800010465e44: mov x0, x19 0.00 : ffff800010465e48: blr x2 0.00 : ffff800010465e4c: ldr x21, [x29, #32] : blk_mq_end_request(): : BUG(); : __blk_mq_end_request(rq, error); : } 0.00 : ffff800010465e50: ldp x19, x20, [sp, #16] 0.00 : ffff800010465e54: ldp x29, x30, [sp], #48 0.00 : ffff800010465e58: ret : blk_mq_need_time_stamp(): : return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator; 0.00 : ffff800010465e5c: ldr x0, [x19] 0.00 : ffff800010465e60: ldr x0, [x0, #8] 0.00 : ffff800010465e64: cbnz x0, ffff800010465dd4 : __blk_mq_end_request(): : u64 now = 0; 0.00 : ffff800010465e68: mov x21, #0x0 // #0 : if (rq->rq_flags & RQF_STATS) { 0.00 : ffff800010465e6c: tbz w1, #17, ffff800010465de4 : blk_mq_poll_stats_start(rq->q); 0.00 : ffff800010465e70: ldr x0, [x19] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010465e74: ldr x1, [x0, #104] : blk_mq_poll_stats_start(): : { : /* : * We don't arm the callback if polling stats are not enabled or the : * callback is already active. : */ : if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 0.00 : ffff800010465e78: tst w1, #0x200000 0.00 : ffff800010465e7c: b.eq ffff800010465e8c // b.none : blk_stat_is_active(q->poll_cb)) 0.00 : ffff800010465e80: ldr x0, [x0, #272] : if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || 0.00 : ffff800010465e84: ldr x1, [x0, #24] 0.00 : ffff800010465e88: cbz x1, ffff800010465eb4 : __blk_mq_end_request(): : blk_stat_add(rq, now); 0.00 : ffff800010465e8c: mov x1, x21 0.00 : ffff800010465e90: mov x0, x19 0.00 : ffff800010465e94: bl ffff80001046b250 0.00 : ffff800010465e98: b ffff800010465de4 : blk_mq_free_request(rq); 11.46 : ffff800010465e9c: mov x0, x19 0.00 : ffff800010465ea0: bl ffff800010465af8 8.06 : ffff800010465ea4: ldr x21, [x29, #32] : blk_mq_end_request(): : } 28.09 : ffff800010465ea8: ldp x19, x20, [sp, #16] 0.04 : ffff800010465eac: ldp x29, x30, [sp], #48 0.00 : ffff800010465eb0: ret : blk_stat_activate_msecs(): : * The timer callback will be called when the window expires. : */ : static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb, : unsigned int msecs) : { : mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs)); 0.00 : ffff800010465eb4: adrp x1, ffff800011907000 0.00 : ffff800010465eb8: add x0, x0, #0x10 0.00 : ffff800010465ebc: ldr x1, [x1, #2432] 0.00 : ffff800010465ec0: add x1, x1, #0x19 0.00 : ffff800010465ec4: bl ffff80001016a270 0.00 : ffff800010465ec8: b ffff800010465e8c : blk_mq_end_request(): : BUG(); 0.00 : ffff800010465ecc: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (2485 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd7fe0 <_raw_spin_trylock>: : _raw_spin_trylock(): : : #endif : : #ifndef CONFIG_INLINE_SPIN_TRYLOCK : int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) : { 0.24 : ffff800010cd7fe0: stp x29, x30, [sp, #-16]! 0.00 : ffff800010cd7fe4: mov x3, x0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 2.64 : ffff800010cd7fe8: mrs x2, sp_el0 : _raw_spin_trylock(): 0.00 : ffff800010cd7fec: mov x29, sp : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.04 : ffff800010cd7ff0: ldr w1, [x2, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010cd7ff4: add w1, w1, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.45 : ffff800010cd7ff8: str w1, [x2, #16] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd7ffc: ldr w0, [x0] : queued_spin_trylock(): : */ : static __always_inline int queued_spin_trylock(struct qspinlock *lock) : { : u32 val = atomic_read(&lock->val); : : if (unlikely(val)) 0.39 : ffff800010cd8000: cbnz w0, ffff800010cd8048 <_raw_spin_trylock+0x68> : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 45.92 : ffff800010cd8004: b ffff800010cd802c <_raw_spin_trylock+0x4c> 0.24 : ffff800010cd8008: b ffff800010cd802c <_raw_spin_trylock+0x4c> : __lse__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, ) : __CMPXCHG_CASE(w, , , 32, ) : __CMPXCHG_CASE(x, , , 64, ) : __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") : __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 0.00 : ffff800010cd800c: mov w1, #0x0 // #0 0.00 : ffff800010cd8010: mov x0, x3 0.00 : ffff800010cd8014: mov w2, #0x1 // #1 0.00 : ffff800010cd8018: mov w4, w1 0.00 : ffff800010cd801c: casa w4, w2, [x3] 48.92 : ffff800010cd8020: mov w0, w4 0.00 : ffff800010cd8024: mov w1, w0 0.00 : ffff800010cd8028: b ffff800010cd8038 <_raw_spin_trylock+0x58> : __ll_sc__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, , , , , K) : __CMPXCHG_CASE(w, , , 32, , , , , K) : __CMPXCHG_CASE( , , , 64, , , , , L) : __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) : __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 0.00 : ffff800010cd802c: mov x0, #0x0 // #0 0.00 : ffff800010cd8030: mov w2, #0x1 // #1 0.00 : ffff800010cd8034: b ffff800010cd87d4 <_raw_read_lock_irqsave+0xfc> : __raw_spin_trylock(): : static inline int __raw_spin_trylock(raw_spinlock_t *lock) : { : preempt_disable(); : if (do_raw_spin_trylock(lock)) { : spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); : return 1; 0.00 : ffff800010cd8038: mov w0, #0x1 // #1 : if (do_raw_spin_trylock(lock)) { 0.00 : ffff800010cd803c: cbnz w1, ffff800010cd8048 <_raw_spin_trylock+0x68> : _raw_spin_trylock(): : return __raw_spin_trylock(lock); : } 0.00 : ffff800010cd8040: ldp x29, x30, [sp], #16 0.00 : ffff800010cd8044: ret : get_current(): 0.16 : ffff800010cd8048: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff800010cd804c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010cd8050: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010cd8054: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010cd8058: cbz x0, ffff800010cd8064 <_raw_spin_trylock+0x84> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd805c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010cd8060: cbnz x0, ffff800010cd8068 <_raw_spin_trylock+0x88> : __raw_spin_trylock(): : } : preempt_enable(); 0.00 : ffff800010cd8064: bl ffff800010cd2e50 : return 0; 0.00 : ffff800010cd8068: mov w0, #0x0 // #0 : _raw_spin_trylock(): 0.00 : ffff800010cd806c: ldp x29, x30, [sp], #16 0.00 : ffff800010cd8070: ret Percent | Source code & Disassembly of vmlinux for cycles (2313 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010713a40 : : iommu_dma_free_iova(): : dma_addr_t iova, size_t size) : { : struct iova_domain *iovad = &cookie->iovad; : : /* The MSI case is only ever cleaning up its most recent allocation */ : if (cookie->type == IOMMU_DMA_MSI_COOKIE) 0.04 : ffff800010713a40: ldr w3, [x0] 0.00 : ffff800010713a44: cmp w3, #0x1 0.00 : ffff800010713a48: b.eq ffff800010713a90 // b.none : { 35.89 : ffff800010713a4c: stp x29, x30, [sp, #-16]! : struct iova_domain *iovad = &cookie->iovad; 0.00 : ffff800010713a50: add x0, x0, #0x8 : { 5.16 : ffff800010713a54: mov x29, sp 6.95 : ffff800010713a58: ldr x3, [x0, #32] : cookie->msi_iova -= size; : else if (cookie->fq_domain) /* non-strict mode */ 8.12 : ffff800010713a5c: ldr x4, [x0, #1888] 0.00 : ffff800010713a60: rbit x3, x3 0.00 : ffff800010713a64: clz x3, x3 0.00 : ffff800010713a68: lsr x1, x1, x3 31.02 : ffff800010713a6c: lsr x2, x2, x3 0.00 : ffff800010713a70: cbz x4, ffff800010713a84 : queue_iova(iovad, iova_pfn(iovad, iova), 0.00 : ffff800010713a74: mov x3, #0x0 // #0 0.00 : ffff800010713a78: bl ffff800010718120 : size >> iova_shift(iovad), 0); : else : free_iova_fast(iovad, iova_pfn(iovad, iova), : size >> iova_shift(iovad)); : } 0.00 : ffff800010713a7c: ldp x29, x30, [sp], #16 0.00 : ffff800010713a80: ret : free_iova_fast(iovad, iova_pfn(iovad, iova), 12.75 : ffff800010713a84: bl ffff800010717cf8 : } 0.09 : ffff800010713a88: ldp x29, x30, [sp], #16 0.00 : ffff800010713a8c: ret : cookie->msi_iova -= size; 0.00 : ffff800010713a90: ldr x1, [x0, #8] 0.00 : ffff800010713a94: sub x2, x1, x2 0.00 : ffff800010713a98: str x2, [x0, #8] 0.00 : ffff800010713a9c: ret Percent | Source code & Disassembly of vmlinux for cycles (2303 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001027df98 : : fput(): : schedule_delayed_work(&delayed_fput_work, 1); : } : } : : void fput(struct file *file) : { 8.59 : ffff80001027df98: stp x29, x30, [sp, #-16]! : fput_many(file, 1); 0.00 : ffff80001027df9c: mov w1, #0x1 // #1 : { 0.00 : ffff80001027dfa0: mov x29, sp : fput_many(file, 1); 0.00 : ffff80001027dfa4: bl ffff80001027ded0 : } 91.41 : ffff80001027dfa8: ldp x29, x30, [sp], #16 0.00 : ffff80001027dfac: ret Percent | Source code & Disassembly of vmlinux for cycles (2337 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001014c630 : : handle_irq_event(): : note_interrupt(desc, retval); : return retval; : } : : irqreturn_t handle_irq_event(struct irq_desc *desc) : { 0.64 : ffff80001014c630: stp x29, x30, [sp, #-48]! 0.00 : ffff80001014c634: mov x29, sp 1.40 : ffff80001014c638: stp x19, x20, [sp, #16] 0.00 : ffff80001014c63c: mov x19, x0 1.96 : ffff80001014c640: str x21, [sp, #32] : irqreturn_t ret; : : desc->istate &= ~IRQS_PENDING; : irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS); : raw_spin_unlock(&desc->lock); 0.00 : ffff80001014c644: add x20, x0, #0xd4 0.00 : ffff80001014c648: mov x0, x20 : desc->istate &= ~IRQS_PENDING; 11.59 : ffff80001014c64c: ldr w1, [x19, #172] : irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS); 1.98 : ffff80001014c650: ldr x2, [x19, #104] : desc->istate &= ~IRQS_PENDING; 0.00 : ffff80001014c654: and w1, w1, #0xfffffdff 3.17 : ffff80001014c658: str w1, [x19, #172] : irqd_set(): : __irqd_to_state(d) &= ~mask; : } : : static inline void irqd_set(struct irq_data *d, unsigned int mask) : { : __irqd_to_state(d) |= mask; 0.98 : ffff80001014c65c: ldr w1, [x2] 0.00 : ffff80001014c660: orr w1, w1, #0x40000 14.74 : ffff80001014c664: str w1, [x2] : handle_irq_event(): : raw_spin_unlock(&desc->lock); 0.00 : ffff80001014c668: bl ffff800010cd7c38 <_raw_spin_unlock> : : ret = handle_irq_event_percpu(desc); 36.06 : ffff80001014c66c: mov x0, x19 0.00 : ffff80001014c670: bl ffff80001014c5a8 0.43 : ffff80001014c674: mov w21, w0 : : raw_spin_lock(&desc->lock); 0.00 : ffff80001014c678: mov x0, x20 0.00 : ffff80001014c67c: bl ffff800010cd80f0 <_raw_spin_lock> : irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS); 13.59 : ffff80001014c680: ldr x2, [x19, #104] : return ret; : } 0.00 : ffff80001014c684: mov w0, w21 : irqd_clear(): : __irqd_to_state(d) &= ~mask; 0.04 : ffff80001014c688: ldr w1, [x2] 0.00 : ffff80001014c68c: and w1, w1, #0xfffbffff 0.00 : ffff80001014c690: str w1, [x2] : handle_irq_event(): 3.44 : ffff80001014c694: ldp x19, x20, [sp, #16] 9.96 : ffff80001014c698: ldr x21, [sp, #32] 0.00 : ffff80001014c69c: ldp x29, x30, [sp], #48 0.00 : ffff80001014c6a0: ret Percent | Source code & Disassembly of vmlinux for cycles (2756 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010087500 : : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010087500: b ffff800010087514 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010087504: nop : __cpu_do_idle(): : : void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); : : static void __cpu_do_idle(void) : { : dsb(sy); 0.00 : ffff800010087508: dsb sy : wfi(); 0.00 : ffff80001008750c: wfi : cpu_do_idle(): : { : if (system_uses_irq_prio_masking()) : __cpu_do_idle_irqprio(); : else : __cpu_do_idle(); : } 0.00 : ffff800010087510: ret : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010087514: adrp x0, ffff800011af2000 0.00 : ffff800010087518: ldr x0, [x0, #1320] : cpu_do_idle(): : if (system_uses_irq_prio_masking()) 0.00 : ffff80001008751c: tbz x0, #42, ffff800010087508 : __cpu_do_idle_irqprio(): : daif_bits = read_sysreg(daif); 0.34 : ffff800010087520: mrs x1, daif : write_sysreg(daif_bits | PSR_I_BIT, daif); 0.00 : ffff800010087524: orr x0, x1, #0x80 0.00 : ffff800010087528: msr daif, x0 : gic_read_pmr(): 0.00 : ffff80001008752c: mrs x0, s3_0_c4_c6_0 : gic_write_pmr(): : return read_sysreg_s(SYS_ICC_PMR_EL1); : } : : static inline void gic_write_pmr(u32 val) : { : write_sysreg_s(val, SYS_ICC_PMR_EL1); 0.00 : ffff800010087530: mov x2, #0xf0 // #240 0.00 : ffff800010087534: msr s3_0_c4_c6_0, x2 : __cpu_do_idle(): : dsb(sy); 0.00 : ffff800010087538: dsb sy : wfi(); 0.00 : ffff80001008753c: wfi : gic_write_pmr(): 0.00 : ffff800010087540: and x0, x0, #0xffffffff 0.00 : ffff800010087544: msr s3_0_c4_c6_0, x0 : __cpu_do_idle_irqprio(): : write_sysreg(daif_bits, daif); 0.00 : ffff800010087548: msr daif, x1 : cpu_do_idle(): : } 99.66 : ffff80001008754c: ret Percent | Source code & Disassembly of vmlinux for cycles (2106 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104706c8 : : part_dec_in_flight(): : part_stat_local_inc(&part_to_disk(part)->part0, in_flight[rw]); : } : : void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw) : { : if (queue_is_mq(q)) 1.70 : ffff8000104706c8: ldr x0, [x0, #48] 0.00 : ffff8000104706cc: cbz x0, ffff8000104706d4 : return; : : part_stat_local_dec(part, in_flight[rw]); : if (part->partno) : part_stat_local_dec(&part_to_disk(part)->part0, in_flight[rw]); : } 98.30 : ffff8000104706d0: ret : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000104706d4: mrs x4, tpidr_el1 : part_dec_in_flight(): : part_stat_local_dec(part, in_flight[rw]); 0.00 : ffff8000104706d8: adrp x3, ffff80001151d000 0.00 : ffff8000104706dc: add x0, x3, #0x18 0.00 : ffff8000104706e0: ldrsw x6, [x0, x4] 0.00 : ffff8000104706e4: adrp x5, ffff800011909000 0.00 : ffff8000104706e8: add x4, x5, #0x928 0.00 : ffff8000104706ec: sxtw x2, w2 0.00 : ffff8000104706f0: add x2, x2, #0x12 0.00 : ffff8000104706f4: ldr x0, [x1, #840] 0.00 : ffff8000104706f8: ldr x4, [x4, x6, lsl #3] 0.00 : ffff8000104706fc: lsl x2, x2, #3 0.00 : ffff800010470700: add x0, x0, x4 0.00 : ffff800010470704: add x0, x0, x2 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010470708: b ffff800010470774 0.00 : ffff80001047070c: b ffff800010470774 : __lse_atomic64_sub(): : : #undef ATOMIC64_FETCH_OP_AND : : static inline void __lse_atomic64_sub(s64 i, atomic64_t *v) : { : asm volatile( 0.00 : ffff800010470710: mov x4, #0x1 // #1 0.00 : ffff800010470714: neg x4, x4 0.00 : ffff800010470718: stadd x4, [x0] : part_dec_in_flight(): : if (part->partno) 0.00 : ffff80001047071c: ldr w0, [x1, #820] 0.00 : ffff800010470720: cbz w0, ffff8000104706d0 : part_to_disk(): : struct lockdep_map lockdep_map; : }; : : static inline struct gendisk *part_to_disk(struct hd_struct *part) : { : if (likely(part)) { 0.00 : ffff800010470724: cbz x1, ffff800010470780 : if (part->partno) : return dev_to_disk(part_to_dev(part)->parent); 0.00 : ffff800010470728: ldr x0, [x1, #104] 0.00 : ffff80001047072c: sub x0, x0, #0x70 : part_dec_in_flight(): : part_stat_local_dec(&part_to_disk(part)->part0, in_flight[rw]); 0.00 : ffff800010470730: add x3, x3, #0x18 0.00 : ffff800010470734: add x5, x5, #0x928 : __my_cpu_offset(): 0.00 : ffff800010470738: mrs x1, tpidr_el1 : part_dec_in_flight(): 0.00 : ffff80001047073c: ldrsw x1, [x3, x1] 0.00 : ffff800010470740: ldr x0, [x0, #912] 0.00 : ffff800010470744: ldr x1, [x5, x1, lsl #3] 0.00 : ffff800010470748: add x0, x0, x1 0.00 : ffff80001047074c: add x2, x0, x2 : arch_static_branch_jump(): 0.00 : ffff800010470750: b ffff800010470768 0.00 : ffff800010470754: b ffff800010470768 : __lse_atomic64_sub(): 0.00 : ffff800010470758: mov x0, #0x1 // #1 0.00 : ffff80001047075c: neg x0, x0 0.00 : ffff800010470760: stadd x0, [x2] : part_dec_in_flight(): : } 0.00 : ffff800010470764: ret : __ll_sc_atomic64_sub(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff800010470768: mov x0, #0x1 // #1 0.00 : ffff80001047076c: b ffff800010471f34 : part_dec_in_flight(): 0.00 : ffff800010470770: ret : __ll_sc_atomic64_sub(): 0.00 : ffff800010470774: mov x4, #0x1 // #1 0.00 : ffff800010470778: b ffff800010471f4c 0.00 : ffff80001047077c: b ffff80001047071c : part_to_disk(): : else : return dev_to_disk(part_to_dev(part)); : } : return NULL; 0.00 : ffff800010470780: mov x0, #0x0 // #0 0.00 : ffff800010470784: b ffff800010470730 Percent | Source code & Disassembly of vmlinux for cycles (2054 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010096ae0 : : smp_send_reschedule(): : trace_ipi_exit_rcuidle(ipi_types[ipinr]); : set_irq_regs(old_regs); : } : : void smp_send_reschedule(int cpu) : { 8.87 : ffff800010096ae0: stp x29, x30, [sp, #-16]! : smp_cross_call(): : __smp_cross_call(target, ipinr); 0.00 : ffff800010096ae4: adrp x1, ffff800011af2000 : get_cpu_mask(): : cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)]; : : static inline const struct cpumask *get_cpu_mask(unsigned int cpu) : { : const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; : p -= cpu / BITS_PER_LONG; 0.00 : ffff800010096ae8: ubfiz x3, x0, #5, #6 0.00 : ffff800010096aec: adrp x2, ffff800010d05000 : smp_send_reschedule(): : { 8.60 : ffff800010096af0: mov x29, sp : smp_cross_call(): : __smp_cross_call(target, ipinr); 0.10 : ffff800010096af4: ldr x4, [x1, #2248] : get_cpu_mask(): 2.55 : ffff800010096af8: add x2, x2, #0x198 0.00 : ffff800010096afc: add x3, x3, #0x20 56.82 : ffff800010096b00: lsr w0, w0, #6 : smp_cross_call(): 19.52 : ffff800010096b04: mov w1, #0x0 // #0 : get_cpu_mask(): 0.19 : ffff800010096b08: sub x0, x3, x0, lsl #3 : smp_cross_call(): 0.00 : ffff800010096b0c: add x0, x2, x0 3.36 : ffff800010096b10: blr x4 : smp_send_reschedule(): : smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); : } 0.00 : ffff800010096b14: ldp x29, x30, [sp], #16 0.00 : ffff800010096b18: ret Percent | Source code & Disassembly of vmlinux for cycles (2002 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010464258 <__blk_mq_free_request>: : __blk_mq_free_request(): : return rq; : } : EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); : : static void __blk_mq_free_request(struct request *rq) : { 0.00 : ffff800010464258: stp x29, x30, [sp, #-64]! 0.00 : ffff80001046425c: mov x29, sp 0.10 : ffff800010464260: stp x19, x20, [sp, #16] 0.00 : ffff800010464264: mov x19, x0 0.05 : ffff800010464268: stp x21, x22, [sp, #32] 16.09 : ffff80001046426c: stp x23, x24, [sp, #48] : struct request_queue *q = rq->q; : struct blk_mq_ctx *ctx = rq->mq_ctx; 0.25 : ffff800010464270: ldp x21, x24, [x0] : struct blk_mq_hw_ctx *hctx = rq->mq_hctx; : const int sched_tag = rq->internal_tag; 1.30 : ffff800010464274: ldr w22, [x0, #36] : struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 0.70 : ffff800010464278: ldr x20, [x0, #16] : blk_pm_mark_last_busy(): : pm_request_resume(q->dev); : } : : static inline void blk_pm_mark_last_busy(struct request *rq) : { : if (rq->q->dev && !(rq->rq_flags & RQF_PM)) 22.43 : ffff80001046427c: ldr x23, [x21, #216] 0.00 : ffff800010464280: cbz x23, ffff80001046428c <__blk_mq_free_request+0x34> 0.00 : ffff800010464284: ldr w0, [x0, #28] 0.00 : ffff800010464288: tbz w0, #15, ffff8000104642e4 <__blk_mq_free_request+0x8c> : __blk_mq_free_request(): : : blk_pm_mark_last_busy(rq); : rq->mq_hctx = NULL; : if (rq->tag != -1) 8.79 : ffff80001046428c: ldr w2, [x19, #32] : rq->mq_hctx = NULL; 0.86 : ffff800010464290: str xzr, [x19, #16] : if (rq->tag != -1) 0.00 : ffff800010464294: cmn w2, #0x1 0.05 : ffff800010464298: b.eq ffff8000104642a8 <__blk_mq_free_request+0x50> // b.none : blk_mq_put_tag(hctx->tags, ctx, rq->tag); 0.75 : ffff80001046429c: ldr x0, [x20, #336] 0.05 : ffff8000104642a0: mov x1, x24 4.39 : ffff8000104642a4: bl ffff80001046a8d0 : if (sched_tag != -1) 11.93 : ffff8000104642a8: cmn w22, #0x1 0.00 : ffff8000104642ac: b.eq ffff8000104642c0 <__blk_mq_free_request+0x68> // b.none : blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag); 0.00 : ffff8000104642b0: ldr x0, [x20, #344] 0.00 : ffff8000104642b4: mov w2, w22 0.00 : ffff8000104642b8: mov x1, x24 0.00 : ffff8000104642bc: bl ffff80001046a8d0 : blk_mq_sched_restart(hctx); 0.35 : ffff8000104642c0: mov x0, x20 0.00 : ffff8000104642c4: bl ffff80001046c980 : blk_queue_exit(q); 11.73 : ffff8000104642c8: mov x0, x21 0.00 : ffff8000104642cc: bl ffff80001045bb30 : } 4.47 : ffff8000104642d0: ldp x19, x20, [sp, #16] 2.84 : ffff8000104642d4: ldp x21, x22, [sp, #32] 3.65 : ffff8000104642d8: ldp x23, x24, [sp, #48] 9.23 : ffff8000104642dc: ldp x29, x30, [sp], #64 0.00 : ffff8000104642e0: ret : pm_runtime_mark_last_busy(): : return !dev->power.no_callbacks; : } : : static inline void pm_runtime_mark_last_busy(struct device *dev) : { : WRITE_ONCE(dev->power.last_busy, ktime_get_mono_fast_ns()); 0.00 : ffff8000104642e4: bl ffff80001016dbe0 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000104642e8: str x0, [x23, #480] 0.00 : ffff8000104642ec: b ffff80001046428c <__blk_mq_free_request+0x34> Percent | Source code & Disassembly of vmlinux for cycles (1999 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104b4d90 : : gic_eoimode1_eoi_irq(): : { : /* : * No need to deactivate an LPI, or an interrupt that : * is is getting forwarded to a vcpu. : */ : if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 83.86 : ffff8000104b4d90: ldr x1, [x0, #8] 0.00 : ffff8000104b4d94: mov w2, #0x1fff // #8191 0.00 : ffff8000104b4d98: cmp w1, w2 0.00 : ffff8000104b4d9c: b.hi ffff8000104b4db8 // b.pmore : irqd_is_forwarded_to_vcpu(): : return __irqd_to_state(d) & IRQD_WAKEUP_ARMED; : } : : static inline bool irqd_is_forwarded_to_vcpu(struct irq_data *d) : { : return __irqd_to_state(d) & IRQD_FORWARDED_TO_VCPU; 0.20 : ffff8000104b4da0: ldr x0, [x0, #16] 0.00 : ffff8000104b4da4: ldr w0, [x0] : gic_eoimode1_eoi_irq(): 0.00 : ffff8000104b4da8: tbnz w0, #20, ffff8000104b4db8 : gic_write_dir(): : isb(); : } : : static __always_inline void gic_write_dir(u32 irq) : { : write_sysreg_s(irq, SYS_ICC_DIR_EL1); 0.25 : ffff8000104b4dac: and x1, x1, #0xffffffff 0.10 : ffff8000104b4db0: msr s3_0_c12_c11_1, x1 : isb(); 0.00 : ffff8000104b4db4: isb : gic_eoimode1_eoi_irq(): : return; : gic_write_dir(gic_irq(d)); : } 15.59 : ffff8000104b4db8: ret Percent | Source code & Disassembly of vmlinux for cycles (1892 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015ded0 : : rcu_dynticks_eqs_enter(): : * Record entry into an extended quiescent state. This is only to be : * called when not already in an extended quiescent state. : */ : static void rcu_dynticks_eqs_enter(void) : { : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.10 : ffff80001015ded0: adrp x0, ffff800011529000 0.00 : ffff80001015ded4: add x0, x0, #0xac0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.05 : ffff80001015ded8: mrs x1, tpidr_el1 : rcu_dynticks_eqs_enter(): 0.20 : ffff80001015dedc: add x0, x0, x1 : /* : * CPUs seeing atomic_add_return() must see prior RCU read-side : * critical sections, and we also must force ordering with the : * next idle sojourn. : */ : seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); 0.05 : ffff80001015dee0: add x2, x0, #0xd8 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001015dee4: b ffff80001015defc 0.05 : ffff80001015dee8: b ffff80001015defc : __lse_atomic_add_return(): : } : : ATOMIC_OP_ADD_RETURN(_relaxed, ) : ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") : ATOMIC_OP_ADD_RETURN(_release, l, "memory") : ATOMIC_OP_ADD_RETURN( , al, "memory") 0.00 : ffff80001015deec: mov w1, #0x2 // #2 0.05 : ffff80001015def0: ldaddal w1, w3, [x2] 99.48 : ffff80001015def4: add w1, w1, w3 : rcu_dynticks_eqs_enter(): : WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && : (seq & RCU_DYNTICK_CTRL_CTR)); : /* Better not have special action (TLB flush) pending! */ : WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && : (seq & RCU_DYNTICK_CTRL_MASK)); : } 0.00 : ffff80001015def8: ret : __ll_sc_atomic_add_return(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff80001015defc: add x0, x0, #0xd8 0.00 : ffff80001015df00: b ffff80001016251c : rcu_dynticks_eqs_enter(): 0.00 : ffff80001015df04: ret Percent | Source code & Disassembly of vmlinux for cycles (1768 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107cb878 : : nvme_error_status(): : return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple); : } : : static blk_status_t nvme_error_status(u16 status) : { : switch (status & 0x7ff) { 32.26 : ffff8000107cb878: and w1, w0, #0x7ff 0.00 : ffff8000107cb87c: cmp w1, #0x180 0.10 : ffff8000107cb880: b.eq ffff8000107cb8f0 // b.none 2.67 : ffff8000107cb884: b.hi ffff8000107cb8bc // b.pmore 21.68 : ffff8000107cb888: cmp w1, #0x21 0.00 : ffff8000107cb88c: b.eq ffff8000107cb910 // b.none 16.67 : ffff8000107cb890: b.ls ffff8000107cb8e0 // b.plast : case NVME_SC_SUCCESS: : return BLK_STS_OK; : case NVME_SC_CAP_EXCEEDED: : return BLK_STS_NOSPC; 0.00 : ffff8000107cb894: mov w0, #0x3 // #3 : switch (status & 0x7ff) { 0.00 : ffff8000107cb898: cmp w1, #0x81 0.00 : ffff8000107cb89c: b.eq ffff8000107cb8b8 // b.none 0.00 : ffff8000107cb8a0: b.ls ffff8000107cb908 // b.plast 0.00 : ffff8000107cb8a4: cmp w1, #0x82 0.00 : ffff8000107cb8a8: b.eq ffff8000107cb910 // b.none : case NVME_SC_APPTAG_CHECK: : case NVME_SC_REFTAG_CHECK: : case NVME_SC_INVALID_PI: : return BLK_STS_PROTECTION; : case NVME_SC_RESERVATION_CONFLICT: : return BLK_STS_NEXUS; 0.00 : ffff8000107cb8ac: mov w0, #0x6 // #6 : switch (status & 0x7ff) { 0.00 : ffff8000107cb8b0: cmp w1, #0x83 0.00 : ffff8000107cb8b4: b.ne ffff8000107cb8d8 // b.any : case NVME_SC_HOST_PATH_ERROR: : return BLK_STS_TRANSPORT; : default: : return BLK_STS_IOERR; : } : } 3.30 : ffff8000107cb8b8: ret : switch (status & 0x7ff) { 0.00 : ffff8000107cb8bc: cmp w1, #0x281 0.00 : ffff8000107cb8c0: b.ls ffff8000107cb920 // b.plast 0.00 : ffff8000107cb8c4: cmp w1, #0x287 0.00 : ffff8000107cb8c8: b.ls ffff8000107cb8f8 // b.plast : return BLK_STS_TRANSPORT; 0.00 : ffff8000107cb8cc: mov w0, #0x4 // #4 : switch (status & 0x7ff) { 0.00 : ffff8000107cb8d0: cmp w1, #0x370 0.00 : ffff8000107cb8d4: b.eq ffff8000107cb8b8 // b.none : return BLK_STS_IOERR; 0.00 : ffff8000107cb8d8: mov w0, #0xa // #10 : } 0.00 : ffff8000107cb8dc: ret : switch (status & 0x7ff) { 1.26 : ffff8000107cb8e0: cmp w1, #0x2 0.00 : ffff8000107cb8e4: b.hi ffff8000107cb944 // b.pmore : return BLK_STS_OK; 22.06 : ffff8000107cb8e8: mov w0, #0x0 // #0 : switch (status & 0x7ff) { 0.00 : ffff8000107cb8ec: cbz w1, ffff8000107cb8b8 : return BLK_STS_NOTSUPP; 0.00 : ffff8000107cb8f0: mov w0, #0x1 // #1 : } 0.00 : ffff8000107cb8f4: ret : switch (status & 0x7ff) { 0.00 : ffff8000107cb8f8: cmp w1, #0x285 0.00 : ffff8000107cb8fc: b.cs ffff8000107cb918 // b.hs, b.nlast : return BLK_STS_PROTECTION; 0.00 : ffff8000107cb900: mov w0, #0x8 // #8 : } 0.00 : ffff8000107cb904: ret : switch (status & 0x7ff) { 0.00 : ffff8000107cb908: cmp w1, #0x80 0.00 : ffff8000107cb90c: b.ne ffff8000107cb8d8 // b.any : return BLK_STS_TARGET; 0.00 : ffff8000107cb910: mov w0, #0x5 // #5 : } 0.00 : ffff8000107cb914: ret : return BLK_STS_MEDIUM; 0.00 : ffff8000107cb918: mov w0, #0x7 // #7 : } 0.00 : ffff8000107cb91c: ret : switch (status & 0x7ff) { 0.00 : ffff8000107cb920: cmp w1, #0x280 0.00 : ffff8000107cb924: b.cs ffff8000107cb918 // b.hs, b.nlast 0.00 : ffff8000107cb928: cmp w1, #0x182 0.00 : ffff8000107cb92c: b.eq ffff8000107cb918 // b.none 0.00 : ffff8000107cb930: b.cc ffff8000107cb900 // b.lo, b.ul, b.last 0.00 : ffff8000107cb934: cmp w1, #0x183 0.00 : ffff8000107cb938: b.ne ffff8000107cb8d8 // b.any : return BLK_STS_NOTSUPP; 0.00 : ffff8000107cb93c: mov w0, #0x1 // #1 0.00 : ffff8000107cb940: b ffff8000107cb8f4 : switch (status & 0x7ff) { 0.00 : ffff8000107cb944: cmp w1, #0xb 0.00 : ffff8000107cb948: b.ne ffff8000107cb8d8 // b.any : return BLK_STS_NOTSUPP; 0.00 : ffff8000107cb94c: mov w0, #0x1 // #1 0.00 : ffff8000107cb950: b ffff8000107cb8f4 Percent | Source code & Disassembly of vmlinux for cycles (1798 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001014b3c8 : : generic_handle_irq(): : * generic_handle_irq - Invoke the handler for a particular irq : * @irq: The irq number to handle : * : */ : int generic_handle_irq(unsigned int irq) : { 13.51 : ffff80001014b3c8: stp x29, x30, [sp, #-16]! : irq_to_desc(): : return radix_tree_lookup(&irq_desc_tree, irq); 0.00 : ffff80001014b3cc: mov w1, w0 0.00 : ffff80001014b3d0: adrp x0, ffff800011921000 0.00 : ffff80001014b3d4: add x0, x0, #0xbe8 : generic_handle_irq(): : { 2.96 : ffff80001014b3d8: mov x29, sp : irq_to_desc(): : return radix_tree_lookup(&irq_desc_tree, irq); 0.00 : ffff80001014b3dc: bl ffff800010cc3e60 : generic_handle_irq(): : struct irq_desc *desc = irq_to_desc(irq); : : if (!desc) 64.39 : ffff80001014b3e0: cbz x0, ffff80001014b3f8 : generic_handle_irq_desc(): : * Architectures call this to let the generic IRQ layer : * handle an interrupt. : */ : static inline void generic_handle_irq_desc(struct irq_desc *desc) : { : desc->handle_irq(desc); 2.95 : ffff80001014b3e4: ldr x1, [x0, #152] 0.00 : ffff80001014b3e8: blr x1 : generic_handle_irq(): : return -EINVAL; : generic_handle_irq_desc(desc); : return 0; 15.47 : ffff80001014b3ec: mov w0, #0x0 // #0 : } 0.72 : ffff80001014b3f0: ldp x29, x30, [sp], #16 0.00 : ffff80001014b3f4: ret : return -EINVAL; 0.00 : ffff80001014b3f8: mov w0, #0xffffffea // #-22 0.00 : ffff80001014b3fc: b ffff80001014b3f0 Percent | Source code & Disassembly of vmlinux for cycles (1751 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010716b60 : : arm_lpae_unmap(): : return __arm_lpae_unmap(data, gather, iova, size, lvl + 1, ptep); : } : : static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, : size_t size, struct iommu_iotlb_gather *gather) : { 9.44 : ffff800010716b60: stp x29, x30, [sp, #-16]! 0.29 : ffff800010716b64: mov x29, sp : struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); : struct io_pgtable_cfg *cfg = &data->iop.cfg; : arm_lpae_iopte *ptep = data->pgd; : long iaext = (long)iova >> cfg->ias; 10.13 : ffff800010716b68: ldur w4, [x0, #-56] 1.08 : ffff800010716b6c: asr x4, x1, x4 : : if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) 0.00 : ffff800010716b70: cbnz x2, ffff800010716b84 0.00 : ffff800010716b74: brk #0x800 : return 0; 0.00 : ffff800010716b78: mov x0, #0x0 // #0 : iaext = ~iaext; : if (WARN_ON(iaext)) : return 0; : : return __arm_lpae_unmap(data, gather, iova, size, data->start_level, ptep); : } 0.00 : ffff800010716b7c: ldp x29, x30, [sp], #16 0.00 : ffff800010716b80: ret : if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) 14.61 : ffff800010716b84: ldur x7, [x0, #-64] 4.57 : ffff800010716b88: and x7, x2, x7 0.00 : ffff800010716b8c: cmp x7, x2 5.12 : ffff800010716b90: b.ne ffff800010716b74 // b.any 9.60 : ffff800010716b94: sub x6, x0, #0x58 : if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 16.98 : ffff800010716b98: ldur x0, [x0, #-72] : iaext = ~iaext; 0.00 : ffff800010716b9c: tst x0, #0x20 0.00 : ffff800010716ba0: cinv x4, x4, ne // ne = any : if (WARN_ON(iaext)) 0.05 : ffff800010716ba4: cbnz x4, ffff800010716bd0 : return __arm_lpae_unmap(data, gather, iova, size, data->start_level, ptep); 1.95 : ffff800010716ba8: ldr w4, [x6, #116] 0.05 : ffff800010716bac: mov x0, x3 15.68 : ffff800010716bb0: ldr x5, [x6, #128] 3.86 : ffff800010716bb4: mov x2, x1 0.00 : ffff800010716bb8: mov x3, x7 0.56 : ffff800010716bbc: mov x1, x0 4.11 : ffff800010716bc0: mov x0, x6 1.48 : ffff800010716bc4: bl ffff800010716718 <__arm_lpae_unmap> : } 0.41 : ffff800010716bc8: ldp x29, x30, [sp], #16 0.00 : ffff800010716bcc: ret : if (WARN_ON(iaext)) 0.00 : ffff800010716bd0: brk #0x800 : return 0; 0.00 : ffff800010716bd4: mov x0, #0x0 // #0 0.00 : ffff800010716bd8: b ffff800010716b7c Percent | Source code & Disassembly of vmlinux for cycles (1706 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010130008 : : autoremove_wake_function(): : } : } : EXPORT_SYMBOL(finish_wait); : : int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key) : { 9.56 : ffff800010130008: stp x29, x30, [sp, #-32]! 0.00 : ffff80001013000c: mov x29, sp 14.08 : ffff800010130010: str x19, [sp, #16] 0.00 : ffff800010130014: mov x19, x0 : int ret = default_wake_function(wq_entry, mode, sync, key); 0.00 : ffff800010130018: bl ffff800010116fe8 : : if (ret) 1.81 : ffff80001013001c: cbz w0, ffff800010130038 : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.12 : ffff800010130020: ldp x3, x2, [x19, #24] : __list_del(): : next->prev = prev; 22.73 : ffff800010130024: str x2, [x3, #8] : autoremove_wake_function(): : list_del_init(&wq_entry->entry); 0.00 : ffff800010130028: add x1, x19, #0x18 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 31.51 : ffff80001013002c: str x3, [x2] 0.06 : ffff800010130030: str x1, [x19, #24] : INIT_LIST_HEAD(): : list->prev = list; 0.12 : ffff800010130034: str x1, [x19, #32] : autoremove_wake_function(): : : return ret; : } 11.40 : ffff800010130038: ldr x19, [sp, #16] 8.61 : ffff80001013003c: ldp x29, x30, [sp], #32 0.00 : ffff800010130040: ret Percent | Source code & Disassembly of vmlinux for cycles (1678 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010115ec0 : : ttwu_do_wakeup(): : } : : /* : * Mark the task runnable and perform wakeup-preemption. : */ : static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags, 7.78 : ffff800010115ec0: stp x29, x30, [sp, #-32]! 0.00 : ffff800010115ec4: mov x29, sp 1.39 : ffff800010115ec8: stp x19, x20, [sp, #16] 0.00 : ffff800010115ecc: mov x20, x1 1.06 : ffff800010115ed0: mov x19, x0 : struct rq_flags *rf) : { : check_preempt_curr(rq, p, wake_flags); 0.18 : ffff800010115ed4: bl ffff800010115e18 : p->state = TASK_RUNNING; : trace_sched_wakeup(p); : : #ifdef CONFIG_SMP : if (p->sched_class->task_woken) { 0.96 : ffff800010115ed8: ldr x0, [x20, #128] : p->state = TASK_RUNNING; 14.73 : ffff800010115edc: str xzr, [x20, #24] : if (p->sched_class->task_woken) { 0.00 : ffff800010115ee0: ldr x2, [x0, #96] 0.00 : ffff800010115ee4: cbz x2, ffff800010115ef4 : /* : * Our task @p is fully woken up and running; so its safe to : * drop the rq->lock, hereafter rq is only used for statistics. : */ : rq_unpin_lock(rq, rf); : p->sched_class->task_woken(rq, p); 0.33 : ffff800010115ee8: mov x1, x20 0.00 : ffff800010115eec: mov x0, x19 0.00 : ffff800010115ef0: blr x2 : rq_repin_lock(rq, rf); : } : : if (rq->idle_stamp) { 24.72 : ffff800010115ef4: ldr x1, [x19, #2816] 0.00 : ffff800010115ef8: cbz x1, ffff800010115f28 : u64 delta = rq_clock(rq) - rq->idle_stamp; 16.30 : ffff800010115efc: ldr x0, [x19, #2400] : update_avg(): : s64 diff = sample - *avg; 2.32 : ffff800010115f00: ldr x3, [x19, #2824] : ttwu_do_wakeup(): : u64 delta = rq_clock(rq) - rq->idle_stamp; 0.00 : ffff800010115f04: sub x0, x0, x1 : u64 max = 2*rq->max_idle_balance_cost; 1.73 : ffff800010115f08: ldr x2, [x19, #2832] : update_avg(): : s64 diff = sample - *avg; 0.00 : ffff800010115f0c: sub x0, x0, x3 : ttwu_do_wakeup(): : u64 max = 2*rq->max_idle_balance_cost; 0.00 : ffff800010115f10: lsl x1, x2, #1 : update_avg(): : *avg += diff >> 3; 0.00 : ffff800010115f14: add x0, x3, x0, asr #3 : ttwu_do_wakeup(): : : update_avg(&rq->avg_idle, delta); : : if (rq->avg_idle > max) 0.28 : ffff800010115f18: cmp x1, x0 0.00 : ffff800010115f1c: b.cs ffff800010115f34 // b.hs, b.nlast : rq->avg_idle = max; : : rq->idle_stamp = 0; 0.00 : ffff800010115f20: str xzr, [x19, #2816] : rq->avg_idle = max; 0.00 : ffff800010115f24: str x1, [x19, #2824] : } : #endif : } 0.17 : ffff800010115f28: ldp x19, x20, [sp, #16] 0.92 : ffff800010115f2c: ldp x29, x30, [sp], #32 0.00 : ffff800010115f30: ret : rq->idle_stamp = 0; 10.95 : ffff800010115f34: str xzr, [x19, #2816] : update_avg(): : *avg += diff >> 3; 16.19 : ffff800010115f38: str x0, [x19, #2824] 0.00 : ffff800010115f3c: b ffff800010115f28 Percent | Source code & Disassembly of vmlinux for cycles (1718 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010087550 : : arch_cpu_idle(): : : /* : * This is our default idle handler. : */ : void arch_cpu_idle(void) : { 0.00 : ffff800010087550: stp x29, x30, [sp, #-16]! 0.00 : ffff800010087554: mov x29, sp : /* : * This should do all the clock switching and wait for interrupt : * tricks : */ : trace_cpu_idle_rcuidle(1, smp_processor_id()); : cpu_do_idle(); 0.00 : ffff800010087558: bl ffff800010087500 : arch_local_irq_enable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff80001008755c: mov x0, #0xe0 // #224 0.00 : ffff800010087560: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 88.40 : ffff800010087564: nop : arch_cpu_idle(): : local_irq_enable(); : trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id()); : } 0.00 : ffff800010087568: ldp x29, x30, [sp], #16 0.00 : ffff80001008756c: ret : arch_local_irq_enable(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" ((unsigned long) GIC_PRIO_IRQON) : : "memory"); : : pmr_sync(); 0.81 : ffff800010087570: dsb sy : arch_cpu_idle(): 10.79 : ffff800010087574: ldp x29, x30, [sp], #16 0.00 : ffff800010087578: ret Percent | Source code & Disassembly of vmlinux for cycles (1830 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010161188 : : rcu_irq_enter(): : * : * If you add or remove a call to rcu_irq_enter(), be sure to test with : * CONFIG_RCU_EQS_DEBUG=y. : */ : void rcu_irq_enter(void) : { 0.05 : ffff800010161188: stp x29, x30, [sp, #-16]! : rcu_nmi_enter_common(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff80001016118c: adrp x0, ffff800011529000 0.00 : ffff800010161190: add x5, x0, #0xac0 : rcu_irq_enter(): : { 0.00 : ffff800010161194: mov x29, sp : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 18.34 : ffff800010161198: mov x2, sp : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001016119c: mrs x1, tpidr_el1 : rcu_nmi_enter_common(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff8000101611a0: add x5, x5, x1 : WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0); 0.00 : ffff8000101611a4: ldr x1, [x5, #208] 0.00 : ffff8000101611a8: tbnz x1, #63, ffff8000101611e4 : rcu_dynticks_curr_cpu_in_eqs(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 44.83 : ffff8000101611ac: add x0, x0, #0xac0 : rcu_nmi_enter_common(): : long incby = 2; 0.00 : ffff8000101611b0: mov x1, #0x2 // #2 : __my_cpu_offset(): 2.38 : ffff8000101611b4: mrs x2, tpidr_el1 : rcu_dynticks_curr_cpu_in_eqs(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff8000101611b8: add x0, x0, x2 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101611bc: ldr w0, [x0, #216] : rcu_nmi_enter_common(): : if (rcu_dynticks_curr_cpu_in_eqs()) { 0.00 : ffff8000101611c0: tbnz w0, #1, ffff8000101611cc : rcu_dynticks_eqs_exit(); 0.05 : ffff8000101611c4: bl ffff80001015df68 : incby = 1; 0.46 : ffff8000101611c8: mov x1, #0x1 // #1 : WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */ 25.63 : ffff8000101611cc: ldr x0, [x5, #208] : __read_once_size(): 7.93 : ffff8000101611d0: ldr w2, [x5, #216] : rcu_nmi_enter_common(): 0.00 : ffff8000101611d4: add x1, x1, x0 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.06 : ffff8000101611d8: str x1, [x5, #208] : rcu_irq_enter(): : lockdep_assert_irqs_disabled(); : rcu_nmi_enter_common(true); : } 0.26 : ffff8000101611dc: ldp x29, x30, [sp], #16 0.00 : ffff8000101611e0: ret : rcu_nmi_enter_common(): : WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0); 0.00 : ffff8000101611e4: brk #0x800 0.00 : ffff8000101611e8: b ffff8000101611ac Percent | Source code & Disassembly of vmlinux for cycles (1556 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012f8f8 <__wake_up_common_lock>: : __wake_up_common_lock(): : return nr_exclusive; : } : : static void __wake_up_common_lock(struct wait_queue_head *wq_head, unsigned int mode, : int nr_exclusive, int wake_flags, void *key) : { 0.13 : ffff80001012f8f8: stp x29, x30, [sp, #-128]! 0.00 : ffff80001012f8fc: mov x29, sp 2.10 : ffff80001012f900: stp x19, x20, [sp, #16] 0.06 : ffff80001012f904: mov w20, w2 19.60 : ffff80001012f908: stp x21, x22, [sp, #32] 0.00 : ffff80001012f90c: mov x22, x4 1.33 : ffff80001012f910: stp x23, x24, [sp, #48] 0.00 : ffff80001012f914: mov w24, w1 0.86 : ffff80001012f918: str x25, [sp, #64] 0.00 : ffff80001012f91c: adrp x25, ffff800011909000 0.00 : ffff80001012f920: add x5, x25, #0x908 0.00 : ffff80001012f924: mov w23, w3 0.93 : ffff80001012f928: ldr x6, [x5] 14.17 : ffff80001012f92c: str x6, [x29, #120] 0.00 : ffff80001012f930: mov x6, #0x0 // #0 0.00 : ffff80001012f934: mov x19, x0 : unsigned long flags; : wait_queue_entry_t bookmark; : : bookmark.flags = 0; 2.37 : ffff80001012f938: str wzr, [x29, #80] : INIT_LIST_HEAD(): : * Initializes the list_head to point to itself. If it is a list header, : * the result is an empty list. : */ : static inline void INIT_LIST_HEAD(struct list_head *list) : { : WRITE_ONCE(list->next, list); 0.00 : ffff80001012f93c: add x0, x29, #0x68 : __wake_up_common_lock(): : bookmark.private = NULL; : bookmark.func = NULL; 0.37 : ffff80001012f940: stp xzr, xzr, [x29, #88] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.58 : ffff80001012f944: str x0, [x29, #104] : INIT_LIST_HEAD(): : list->prev = list; 16.55 : ffff80001012f948: str x0, [x29, #112] 0.00 : ffff80001012f94c: nop : __wake_up_common_lock(): : INIT_LIST_HEAD(&bookmark.entry); : : do { : spin_lock_irqsave(&wq_head->lock, flags); 0.76 : ffff80001012f950: mov x0, x19 0.00 : ffff80001012f954: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : nr_exclusive = __wake_up_common(wq_head, mode, nr_exclusive, 0.00 : ffff80001012f958: mov w2, w20 0.00 : ffff80001012f95c: add x5, x29, #0x50 0.00 : ffff80001012f960: mov x4, x22 0.00 : ffff80001012f964: mov w3, w23 : spin_lock_irqsave(&wq_head->lock, flags); 0.00 : ffff80001012f968: mov x21, x0 : nr_exclusive = __wake_up_common(wq_head, mode, nr_exclusive, 15.06 : ffff80001012f96c: mov w1, w24 0.00 : ffff80001012f970: mov x0, x19 0.00 : ffff80001012f974: bl ffff80001012f7a8 <__wake_up_common> : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 16.56 : ffff80001012f978: mov x1, x21 : __wake_up_common_lock(): 0.00 : ffff80001012f97c: mov w20, w0 : spin_unlock_irqrestore(): 0.00 : ffff80001012f980: mov x0, x19 0.00 : ffff80001012f984: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : __wake_up_common_lock(): : wake_flags, key, &bookmark); : spin_unlock_irqrestore(&wq_head->lock, flags); : } while (bookmark.flags & WQ_FLAG_BOOKMARK); 0.83 : ffff80001012f988: ldr w0, [x29, #80] 0.00 : ffff80001012f98c: tbnz w0, #2, ffff80001012f950 <__wake_up_common_lock+0x58> : } 3.00 : ffff80001012f990: add x25, x25, #0x908 0.13 : ffff80001012f994: ldr x1, [x29, #120] 0.00 : ffff80001012f998: ldr x0, [x25] 0.00 : ffff80001012f99c: eor x0, x1, x0 0.00 : ffff80001012f9a0: cbnz x0, ffff80001012f9bc <__wake_up_common_lock+0xc4> 3.44 : ffff80001012f9a4: ldp x19, x20, [sp, #16] 0.63 : ffff80001012f9a8: ldp x21, x22, [sp, #32] 0.00 : ffff80001012f9ac: ldp x23, x24, [sp, #48] 0.00 : ffff80001012f9b0: ldr x25, [sp, #64] 0.32 : ffff80001012f9b4: ldp x29, x30, [sp], #128 0.20 : ffff80001012f9b8: ret 0.00 : ffff80001012f9bc: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (1535 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cb80c0 <__memset>: : memset(): : tmp3 .req x9 : : .weak memset : SYM_FUNC_START_ALIAS(__memset) : SYM_FUNC_START_PI(memset) : mov dst, dstin /* Preserve return value. */ 0.00 : ffff800010cb80c0: mov x8, x0 : and A_lw, val, #255 0.00 : ffff800010cb80c4: and w7, w1, #0xff : orr A_lw, A_lw, A_lw, lsl #8 0.00 : ffff800010cb80c8: orr w7, w7, w7, lsl #8 : orr A_lw, A_lw, A_lw, lsl #16 0.00 : ffff800010cb80cc: orr w7, w7, w7, lsl #16 : orr A_l, A_l, A_l, lsl #32 0.00 : ffff800010cb80d0: orr x7, x7, x7, lsl #32 : : cmp count, #15 0.00 : ffff800010cb80d4: cmp x2, #0xf : b.hi .Lover16_proc 0.00 : ffff800010cb80d8: b.hi ffff800010cb8100 <__memset+0x40> // b.pmore : /*All store maybe are non-aligned..*/ : tbz count, #3, 1f 0.00 : ffff800010cb80dc: tbz w2, #3, ffff800010cb80e4 <__memset+0x24> : str A_l, [dst], #8 0.00 : ffff800010cb80e0: str x7, [x8], #8 : 1: : tbz count, #2, 2f 0.00 : ffff800010cb80e4: tbz w2, #2, ffff800010cb80ec <__memset+0x2c> : str A_lw, [dst], #4 0.00 : ffff800010cb80e8: str w7, [x8], #4 : 2: : tbz count, #1, 3f 0.00 : ffff800010cb80ec: tbz w2, #1, ffff800010cb80f4 <__memset+0x34> : strh A_lw, [dst], #2 0.00 : ffff800010cb80f0: strh w7, [x8], #2 : 3: : tbz count, #0, 4f 0.00 : ffff800010cb80f4: tbz w2, #0, ffff800010cb80fc <__memset+0x3c> : strb A_lw, [dst] 0.00 : ffff800010cb80f8: strb w7, [x8] : 4: : ret 0.00 : ffff800010cb80fc: ret : : .Lover16_proc: : /*Whether the start address is aligned with 16.*/ : neg tmp2, dst 0.00 : ffff800010cb8100: neg x4, x8 : ands tmp2, tmp2, #15 0.00 : ffff800010cb8104: ands x4, x4, #0xf : b.eq .Laligned 0.00 : ffff800010cb8108: b.eq ffff800010cb8118 <__memset+0x58> // b.none : /* : * The count is not less than 16, we can use stp to store the start 16 bytes, : * then adjust the dst aligned with 16.This process will make the current : * memory address at alignment boundary. : */ : stp A_l, A_l, [dst] /*non-aligned store..*/ 0.00 : ffff800010cb810c: stp x7, x7, [x8] : /*make the dst aligned..*/ : sub count, count, tmp2 0.00 : ffff800010cb8110: sub x2, x2, x4 : add dst, dst, tmp2 0.00 : ffff800010cb8114: add x8, x8, x4 : : .Laligned: : cbz A_l, .Lzero_mem 0.13 : ffff800010cb8118: cbz x7, ffff800010cb81b0 <__memset+0xf0> : : .Ltail_maybe_long: : cmp count, #64 0.00 : ffff800010cb811c: cmp x2, #0x40 : b.ge .Lnot_short 0.00 : ffff800010cb8120: b.ge ffff800010cb8180 <__memset+0xc0> // b.tcont : .Ltail63: : ands tmp1, count, #0x30 0.00 : ffff800010cb8124: ands x3, x2, #0x30 : b.eq 3f 0.00 : ffff800010cb8128: b.eq ffff800010cb8144 <__memset+0x84> // b.none : cmp tmp1w, #0x20 0.00 : ffff800010cb812c: cmp w3, #0x20 : b.eq 1f 0.00 : ffff800010cb8130: b.eq ffff800010cb813c <__memset+0x7c> // b.none : b.lt 2f 0.00 : ffff800010cb8134: b.lt ffff800010cb8140 <__memset+0x80> // b.tstop : stp A_l, A_l, [dst], #16 0.00 : ffff800010cb8138: stp x7, x7, [x8], #16 : 1: : stp A_l, A_l, [dst], #16 0.00 : ffff800010cb813c: stp x7, x7, [x8], #16 : 2: : stp A_l, A_l, [dst], #16 0.00 : ffff800010cb8140: stp x7, x7, [x8], #16 : /* : * The last store length is less than 16,use stp to write last 16 bytes. : * It will lead some bytes written twice and the access is non-aligned. : */ : 3: : ands count, count, #15 0.00 : ffff800010cb8144: ands x2, x2, #0xf : cbz count, 4f 0.00 : ffff800010cb8148: cbz x2, ffff800010cb8154 <__memset+0x94> : add dst, dst, count 0.00 : ffff800010cb814c: add x8, x8, x2 : stp A_l, A_l, [dst, #-16] /* Repeat some/all of last store. */ 0.00 : ffff800010cb8150: stp x7, x7, [x8, #-16] : 4: : ret 0.00 : ffff800010cb8154: ret 0.00 : ffff800010cb8158: nop 0.00 : ffff800010cb815c: nop 0.00 : ffff800010cb8160: nop 0.00 : ffff800010cb8164: nop 0.00 : ffff800010cb8168: nop 0.00 : ffff800010cb816c: nop 0.00 : ffff800010cb8170: nop 0.00 : ffff800010cb8174: nop 0.00 : ffff800010cb8178: nop 0.00 : ffff800010cb817c: nop : * Critical loop. Start at a new cache line boundary. Assuming : * 64 bytes per line, this ensures the entire loop is in one line. : */ : .p2align L1_CACHE_SHIFT : .Lnot_short: : sub dst, dst, #16/* Pre-bias. */ 0.00 : ffff800010cb8180: sub x8, x8, #0x10 : sub count, count, #64 0.00 : ffff800010cb8184: sub x2, x2, #0x40 : 1: : stp A_l, A_l, [dst, #16] 0.00 : ffff800010cb8188: stp x7, x7, [x8, #16] : stp A_l, A_l, [dst, #32] 0.00 : ffff800010cb818c: stp x7, x7, [x8, #32] : stp A_l, A_l, [dst, #48] 0.00 : ffff800010cb8190: stp x7, x7, [x8, #48] : stp A_l, A_l, [dst, #64]! 0.00 : ffff800010cb8194: stp x7, x7, [x8, #64]! : subs count, count, #64 0.00 : ffff800010cb8198: subs x2, x2, #0x40 : b.ge 1b 0.00 : ffff800010cb819c: b.ge ffff800010cb8188 <__memset+0xc8> // b.tcont : tst count, #0x3f 0.00 : ffff800010cb81a0: tst x2, #0x3f : add dst, dst, #16 0.00 : ffff800010cb81a4: add x8, x8, #0x10 : b.ne .Ltail63 0.00 : ffff800010cb81a8: b.ne ffff800010cb8124 <__memset+0x64> // b.any : .Lexitfunc: : ret 0.00 : ffff800010cb81ac: ret : /* : * For zeroing memory, check to see if we can use the ZVA feature to : * zero entire 'cache' lines. : */ : .Lzero_mem: : cmp count, #63 0.00 : ffff800010cb81b0: cmp x2, #0x3f : b.le .Ltail63 0.00 : ffff800010cb81b4: b.le ffff800010cb8124 <__memset+0x64> : /* : * For zeroing small amounts of memory, it's not worth setting up : * the line-clear code. : */ : cmp count, #128 0.00 : ffff800010cb81b8: cmp x2, #0x80 : b.lt .Lnot_short /*count is at least 128 bytes*/ 0.00 : ffff800010cb81bc: b.lt ffff800010cb8180 <__memset+0xc0> // b.tstop : : mrs tmp1, dczid_el0 0.00 : ffff800010cb81c0: mrs x3, dczid_el0 : tbnz tmp1, #4, .Lnot_short 0.13 : ffff800010cb81c4: tbnz w3, #4, ffff800010cb8180 <__memset+0xc0> : mov tmp3w, #4 0.00 : ffff800010cb81c8: mov w9, #0x4 // #4 : and zva_len, tmp1w, #15 /* Safety: other bits reserved. */ 0.00 : ffff800010cb81cc: and w5, w3, #0xf : lsl zva_len, tmp3w, zva_len 0.00 : ffff800010cb81d0: lsl w5, w9, w5 : : ands tmp3w, zva_len, #63 0.00 : ffff800010cb81d4: ands w9, w5, #0x3f : /* : * ensure the zva_len is not less than 64. : * It is not meaningful to use ZVA if the block size is less than 64. : */ : b.ne .Lnot_short 0.00 : ffff800010cb81d8: b.ne ffff800010cb8180 <__memset+0xc0> // b.any : .Lzero_by_line: : /* : * Compute how far we need to go to become suitably aligned. We're : * already at quad-word alignment. : */ : cmp count, zva_len_x 0.00 : ffff800010cb81dc: cmp x2, x5 : b.lt .Lnot_short /* Not enough to reach alignment. */ 0.00 : ffff800010cb81e0: b.lt ffff800010cb8180 <__memset+0xc0> // b.tstop : sub zva_bits_x, zva_len_x, #1 0.00 : ffff800010cb81e4: sub x6, x5, #0x1 : neg tmp2, dst 0.00 : ffff800010cb81e8: neg x4, x8 : ands tmp2, tmp2, zva_bits_x 0.00 : ffff800010cb81ec: ands x4, x4, x6 : b.eq 2f /* Already aligned. */ 0.00 : ffff800010cb81f0: b.eq ffff800010cb8228 <__memset+0x168> // b.none : /* Not aligned, check that there's enough to copy after alignment.*/ : sub tmp1, count, tmp2 0.00 : ffff800010cb81f4: sub x3, x2, x4 : /* : * grantee the remain length to be ZVA is bigger than 64, : * avoid to make the 2f's process over mem range.*/ : cmp tmp1, #64 0.00 : ffff800010cb81f8: cmp x3, #0x40 : ccmp tmp1, zva_len_x, #8, ge /* NZCV=0b1000 */ 0.00 : ffff800010cb81fc: ccmp x3, x5, #0x8, ge // ge = tcont : b.lt .Lnot_short 0.00 : ffff800010cb8200: b.lt ffff800010cb8180 <__memset+0xc0> // b.tstop : /* : * We know that there's at least 64 bytes to zero and that it's safe : * to overrun by 64 bytes. : */ : mov count, tmp1 0.00 : ffff800010cb8204: mov x2, x3 : 1: : stp A_l, A_l, [dst] 0.00 : ffff800010cb8208: stp x7, x7, [x8] : stp A_l, A_l, [dst, #16] 0.00 : ffff800010cb820c: stp x7, x7, [x8, #16] : stp A_l, A_l, [dst, #32] 0.00 : ffff800010cb8210: stp x7, x7, [x8, #32] : subs tmp2, tmp2, #64 0.00 : ffff800010cb8214: subs x4, x4, #0x40 : stp A_l, A_l, [dst, #48] 0.00 : ffff800010cb8218: stp x7, x7, [x8, #48] : add dst, dst, #64 0.00 : ffff800010cb821c: add x8, x8, #0x40 : b.ge 1b 0.00 : ffff800010cb8220: b.ge ffff800010cb8208 <__memset+0x148> // b.tcont : /* We've overrun a bit, so adjust dst downwards.*/ : add dst, dst, tmp2 0.00 : ffff800010cb8224: add x8, x8, x4 : 2: : sub count, count, zva_len_x 0.07 : ffff800010cb8228: sub x2, x2, x5 : 3: : dc zva, dst 0.20 : ffff800010cb822c: dc zva, x8 : add dst, dst, zva_len_x 99.41 : ffff800010cb8230: add x8, x8, x5 : subs count, count, zva_len_x 0.00 : ffff800010cb8234: subs x2, x2, x5 : b.ge 3b 0.00 : ffff800010cb8238: b.ge ffff800010cb822c <__memset+0x16c> // b.tcont : ands count, count, zva_bits_x 0.00 : ffff800010cb823c: ands x2, x2, x6 : b.ne .Ltail_maybe_long 0.06 : ffff800010cb8240: b.ne ffff800010cb811c <__memset+0x5c> // b.any : ret 0.00 : ffff800010cb8244: ret Percent | Source code & Disassembly of vmlinux for cycles (1512 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010711750 : : __fls(): : * : * Undefined if no set bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __fls(unsigned long word) : { : return (sizeof(word) * 8) - 1 - __builtin_clzl(word); 6.15 : ffff800010711750: clz x3, x2 0.00 : ffff800010711754: mov x2, #0x3f // #63 0.00 : ffff800010711758: sub x2, x2, x3 : iommu_pgsize(): : { : unsigned int pgsize_idx; : size_t pgsize; : : /* Max page size that still fits into 'size' */ : pgsize_idx = __fls(size); 0.00 : ffff80001071175c: mov w3, w2 : : /* need to consider alignment requirements ? */ : if (likely(addr_merge)) { 0.46 : ffff800010711760: cbz x1, ffff800010711774 : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 3.82 : ffff800010711764: rbit x1, x1 1.54 : ffff800010711768: clz x1, x1 : iommu_pgsize(): : /* Max page size allowed by address */ : unsigned int align_pgsize_idx = __ffs(addr_merge); : pgsize_idx = min(pgsize_idx, align_pgsize_idx); 2.68 : ffff80001071176c: cmp w2, w1 0.20 : ffff800010711770: csel w3, w2, w1, ls // ls = plast : } : : /* build a mask of acceptable page sizes */ : pgsize = (1UL << (pgsize_idx + 1)) - 1; 10.43 : ffff800010711774: add w3, w3, #0x1 : : /* throw away page sizes not supported by the hardware */ : pgsize &= domain->pgsize_bitmap; 0.33 : ffff800010711778: mov x1, #0xffffffffffffffff // #-1 3.47 : ffff80001071177c: lsl x1, x1, x3 : : /* make sure we're still sane */ : BUG_ON(!pgsize); 0.12 : ffff800010711780: bics x1, x0, x1 1.44 : ffff800010711784: b.eq ffff8000107117a0 // b.none : __fls(): 0.40 : ffff800010711788: clz x1, x1 31.40 : ffff80001071178c: mov x0, #0x3f // #63 1.48 : ffff800010711790: sub x1, x0, x1 : iommu_pgsize(): : : /* pick the biggest page */ : pgsize_idx = __fls(pgsize); : pgsize = 1UL << pgsize_idx; 1.92 : ffff800010711794: mov x0, #0x1 // #1 : : return pgsize; : } 32.30 : ffff800010711798: lsl x0, x0, x1 1.86 : ffff80001071179c: ret : BUG_ON(!pgsize); 0.00 : ffff8000107117a0: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (1433 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011ce78 <__enqueue_entity>: : __enqueue_entity(): : /* : * Enqueue an entity into the rb-tree: : */ : static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) : { : struct rb_node **link = &cfs_rq->tasks_timeline.rb_root.rb_node; 7.24 : ffff80001011ce78: add x7, x0, #0x38 : { 9.93 : ffff80001011ce7c: stp x29, x30, [sp, #-16]! : struct rb_node **link = &cfs_rq->tasks_timeline.rb_root.rb_node; 0.00 : ffff80001011ce80: mov x4, x7 : { 0.14 : ffff80001011ce84: mov x8, x0 0.14 : ffff80001011ce88: mov x29, sp : struct rb_node *parent = NULL; : struct sched_entity *entry; : bool leftmost = true; 9.67 : ffff80001011ce8c: mov w6, #0x1 // #1 : struct rb_node *parent = NULL; 0.21 : ffff80001011ce90: mov x3, #0x0 // #0 : : /* : * Find the right place in the rbtree: : */ : while (*link) { 6.00 : ffff80001011ce94: ldr x2, [x4] 0.07 : ffff80001011ce98: cbz x2, ffff80001011cec4 <__enqueue_entity+0x4c> 0.00 : ffff80001011ce9c: nop : entity_before(): : return (s64)(a->vruntime - b->vruntime) < 0; 0.00 : ffff80001011cea0: ldr x3, [x1, #88] : __enqueue_entity(): : * the same key stay together. : */ : if (entity_before(se, entry)) { : link = &parent->rb_left; : } else { : link = &parent->rb_right; 0.00 : ffff80001011cea4: add x4, x2, #0x8 : entity_before(): : return (s64)(a->vruntime - b->vruntime) < 0; 0.00 : ffff80001011cea8: ldr x5, [x2, #64] 0.00 : ffff80001011ceac: sub x3, x3, x5 : __enqueue_entity(): : if (entity_before(se, entry)) { 0.00 : ffff80001011ceb0: tbnz x3, #63, ffff80001011ceec <__enqueue_entity+0x74> : leftmost = false; 0.00 : ffff80001011ceb4: mov x3, x2 0.00 : ffff80001011ceb8: mov w6, #0x0 // #0 : while (*link) { 0.00 : ffff80001011cebc: ldr x2, [x4] 0.00 : ffff80001011cec0: cbnz x2, ffff80001011cea0 <__enqueue_entity+0x28> : } : } : : rb_link_node(&se->run_node, parent, link); 6.86 : ffff80001011cec4: add x0, x1, #0x18 : rb_link_node(): : struct rb_root *root); : : static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, : struct rb_node **rb_link) : { : node->__rb_parent_color = (unsigned long)parent; 9.93 : ffff80001011cec8: str x3, [x1, #24] : node->rb_left = node->rb_right = NULL; 10.85 : ffff80001011cecc: stp xzr, xzr, [x0, #8] : : *rb_link = node; 7.36 : ffff80001011ced0: str x0, [x4] : rb_insert_color_cached(): : : static inline void rb_insert_color_cached(struct rb_node *node, : struct rb_root_cached *root, : bool leftmost) : { : if (leftmost) 0.00 : ffff80001011ced4: cbz w6, ffff80001011cedc <__enqueue_entity+0x64> : root->rb_leftmost = node; 10.36 : ffff80001011ced8: str x0, [x8, #64] : rb_insert_color(node, &root->rb_root); 0.07 : ffff80001011cedc: mov x1, x7 0.00 : ffff80001011cee0: bl ffff800010cc4ca0 : __enqueue_entity(): : rb_insert_color_cached(&se->run_node, : &cfs_rq->tasks_timeline, leftmost); : } 20.98 : ffff80001011cee4: ldp x29, x30, [sp], #16 0.21 : ffff80001011cee8: ret : link = &parent->rb_left; 0.00 : ffff80001011ceec: add x4, x2, #0x10 : leftmost = false; 0.00 : ffff80001011cef0: mov x3, x2 0.00 : ffff80001011cef4: b ffff80001011cebc <__enqueue_entity+0x44> Percent | Source code & Disassembly of vmlinux for cycles (1359 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010456a58 : : bio_free(): : bio_integrity_free(bio); : } : EXPORT_SYMBOL(bio_uninit); : : static void bio_free(struct bio *bio) : { 0.96 : ffff800010456a58: stp x29, x30, [sp, #-32]! 0.00 : ffff800010456a5c: mov x29, sp 1.54 : ffff800010456a60: stp x19, x20, [sp, #16] 0.00 : ffff800010456a64: mov x19, x0 : struct bio_set *bs = bio->bi_pool; 26.73 : ffff800010456a68: ldr x20, [x0, #112] : void *p; : : bio_uninit(bio); 0.00 : ffff800010456a6c: bl ffff800010456448 : : if (bs) { 3.60 : ffff800010456a70: cbz x20, ffff800010456aa4 : bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio)); 27.31 : ffff800010456a74: ldr x1, [x19, #104] 0.00 : ffff800010456a78: add x0, x20, #0x58 8.47 : ffff800010456a7c: ldrh w2, [x19, #20] 0.00 : ffff800010456a80: lsr w2, w2, #13 0.00 : ffff800010456a84: bl ffff8000104569f8 : : /* : * If we have front padding, adjust the bio pointer before freeing : */ : p = bio; : p -= bs->front_pad; 29.03 : ffff800010456a88: ldr w0, [x20, #8] : : mempool_free(p, &bs->bio_pool); 0.00 : ffff800010456a8c: add x1, x20, #0x10 0.00 : ffff800010456a90: sub x0, x19, x0 0.00 : ffff800010456a94: bl ffff8000101d7548 : } else { : /* Bio was allocated by bio_kmalloc() */ : kfree(bio); : } : } 0.14 : ffff800010456a98: ldp x19, x20, [sp, #16] 2.15 : ffff800010456a9c: ldp x29, x30, [sp], #32 0.08 : ffff800010456aa0: ret : kfree(bio); 0.00 : ffff800010456aa4: mov x0, x19 0.00 : ffff800010456aa8: bl ffff800010253890 : } 0.00 : ffff800010456aac: ldp x19, x20, [sp, #16] 0.00 : ffff800010456ab0: ldp x29, x30, [sp], #32 0.00 : ffff800010456ab4: ret Percent | Source code & Disassembly of vmlinux for cycles (2404 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd7c70 <_raw_spin_unlock_irq>: : _raw_spin_unlock_irq(): : EXPORT_SYMBOL(_raw_spin_unlock_irqrestore); : #endif : : #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ : void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) : { 6.37 : ffff800010cd7c70: stp x29, x30, [sp, #-16]! : queued_spin_unlock(): : static __always_inline void queued_spin_unlock(struct qspinlock *lock) : { : /* : * unlock() needs release semantics: : */ : smp_store_release(&lock->locked, 0); 0.00 : ffff800010cd7c74: mov w1, #0x0 // #0 : _raw_spin_unlock_irq(): 0.00 : ffff800010cd7c78: mov x29, sp : queued_spin_unlock(): 0.13 : ffff800010cd7c7c: stlrb w1, [x0] : arch_local_irq_enable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 3.35 : ffff800010cd7c80: mov x0, #0xe0 // #224 6.50 : ffff800010cd7c84: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 16.07 : ffff800010cd7c88: nop : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010cd7c8c: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010cd7c90: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010cd7c94: sub x0, x0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 12.58 : ffff800010cd7c98: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010cd7c9c: cbz x0, ffff800010cd7ca8 <_raw_spin_unlock_irq+0x38> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd7ca0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010cd7ca4: cbnz x0, ffff800010cd7cac <_raw_spin_unlock_irq+0x3c> : __raw_spin_unlock_irq(): : static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock) : { : spin_release(&lock->dep_map, _RET_IP_); : do_raw_spin_unlock(lock); : local_irq_enable(); : preempt_enable(); 0.00 : ffff800010cd7ca8: bl ffff800010cd2e50 : _raw_spin_unlock_irq(): : __raw_spin_unlock_irq(lock); : } 1.21 : ffff800010cd7cac: ldp x29, x30, [sp], #16 0.00 : ffff800010cd7cb0: ret : arch_local_irq_enable(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" ((unsigned long) GIC_PRIO_IRQON) : : "memory"); : : pmr_sync(); 0.79 : ffff800010cd7cb4: dsb sy 52.99 : ffff800010cd7cb8: b ffff800010cd7c8c <_raw_spin_unlock_irq+0x1c> Percent | Source code & Disassembly of vmlinux for cycles (1785 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010087c40 <__switch_to>: : __switch_to(): : /* : * Thread switching. : */ : __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, : struct task_struct *next) : { 1.34 : ffff800010087c40: stp x29, x30, [sp, #-48]! 0.00 : ffff800010087c44: mov x29, sp 3.95 : ffff800010087c48: stp x19, x20, [sp, #16] 0.00 : ffff800010087c4c: mov x19, x1 0.00 : ffff800010087c50: mov x20, x0 0.50 : ffff800010087c54: str x21, [sp, #32] : struct task_struct *last; : : fpsimd_thread_switch(next); 0.00 : ffff800010087c58: mov x0, x1 0.00 : ffff800010087c5c: bl ffff800010085bf8 : tls_preserve_current_state(): : *task_user_tls(current) = read_sysreg(tpidr_el0); 0.06 : ffff800010087c60: mrs x1, tpidr_el0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.17 : ffff800010087c64: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.67 : ffff800010087c68: ldr x3, [x0] : tls_preserve_current_state(): 0.00 : ffff800010087c6c: add x2, x0, #0xa38 0.00 : ffff800010087c70: add x0, x0, #0xa30 0.00 : ffff800010087c74: tst w3, #0x400000 1.10 : ffff800010087c78: csel x0, x0, x2, eq // eq = none 0.00 : ffff800010087c7c: str x1, [x0] : test_bit(): 8.19 : ffff800010087c80: ldr x0, [x19] : tls_thread_switch(): : if (is_compat_thread(task_thread_info(next))) 0.00 : ffff800010087c84: tst w0, #0x400000 0.00 : ffff800010087c88: b.eq ffff800010087e04 <__switch_to+0x1c4> // b.none : write_sysreg(next->thread.uw.tp_value, tpidrro_el0); 0.00 : ffff800010087c8c: ldr x0, [x19, #2608] 0.00 : ffff800010087c90: msr tpidrro_el0, x0 : test_bit(): 1.98 : ffff800010087c94: ldr x0, [x19] : tls_thread_switch(): : write_sysreg(*task_user_tls(next), tpidr_el0); 0.00 : ffff800010087c98: tst w0, #0x400000 0.00 : ffff800010087c9c: b.ne ffff800010087e20 <__switch_to+0x1e0> // b.any 1.68 : ffff800010087ca0: ldr x0, [x19, #2608] 0.69 : ffff800010087ca4: msr tpidr_el0, x0 : __switch_to(): : tls_thread_switch(next); : hw_breakpoint_thread_switch(next); 0.00 : ffff800010087ca8: mov x0, x19 0.00 : ffff800010087cac: bl ffff80001009dc80 : entry_task_switch(): : __this_cpu_write(__entry_task, next); 0.06 : ffff800010087cb0: adrp x1, ffff80001151d000 0.00 : ffff800010087cb4: add x1, x1, #0x298 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010087cb8: mrs x2, tpidr_el1 : entry_task_switch(): 0.00 : ffff800010087cbc: str x19, [x1, x2] : __switch_to(): : contextidr_thread_switch(next); : entry_task_switch(next); : uao_thread_switch(next); 0.00 : ffff800010087cc0: mov x0, x19 0.00 : ffff800010087cc4: bl ffff800010087c20 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.62 : ffff800010087cc8: b ffff800010087df0 <__switch_to+0x1b0> : arch_static_branch(): : asm_volatile_goto( 4.40 : ffff800010087ccc: nop : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff800010087cd0: b ffff800010087d14 <__switch_to+0xd4> : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010087cd4: nop : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff800010087cd8: b ffff800010087ddc <__switch_to+0x19c> : arch_static_branch(): : asm_volatile_goto( 0.68 : ffff800010087cdc: nop : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff800010087ce0: b ffff800010087d64 <__switch_to+0x124> : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010087ce4: nop : ssbs_thread_switch(): : if (unlikely(next->flags & PF_KTHREAD)) 0.00 : ffff800010087ce8: ldr w0, [x19, #44] : task_stack_page(): : * try_get_task_stack() instead. task_stack_page will return a pointer : * that could get freed out from under you. : */ : static inline void *task_stack_page(const struct task_struct *task) : { : return task->stack; 5.78 : ffff800010087cec: ldr x21, [x19, #32] : ssbs_thread_switch(): 0.00 : ffff800010087cf0: tbz w0, #21, ffff800010087d8c <__switch_to+0x14c> : __switch_to(): : * Complete any pending TLB or cache maintenance on this CPU in case : * the thread migrates to a different CPU. : * This full barrier is also required by the membarrier system : * call. : */ : dsb(ish); 0.17 : ffff800010087cf4: dsb ish : : /* the actual thread switch */ : last = cpu_switch_to(prev, next); 61.05 : ffff800010087cf8: mov x1, x19 0.00 : ffff800010087cfc: mov x0, x20 0.00 : ffff800010087d00: bl ffff800010084e74 : : return last; : } 0.06 : ffff800010087d04: ldr x21, [sp, #32] 3.58 : ffff800010087d08: ldp x19, x20, [sp, #16] 1.56 : ffff800010087d0c: ldp x29, x30, [sp], #48 0.00 : ffff800010087d10: ret : test_bit(): 0.00 : ffff800010087d14: adrp x0, ffff800011af2000 0.00 : ffff800010087d18: ldr x0, [x0, #1320] : system_supports_address_auth(): : cpus_have_const_cap(ARM64_HAS_CNP); : } : : static inline bool system_supports_address_auth(void) : { : return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) && 0.00 : ffff800010087d1c: tbz x0, #39, ffff800010087cd8 <__switch_to+0x98> : ptrauth_keys_switch(): : } while (0) : : static inline void ptrauth_keys_switch(struct ptrauth_keys *keys) : { : if (system_supports_address_auth()) { : __ptrauth_key_install(APIA, keys->apia); 0.00 : ffff800010087d20: ldr x0, [x19, #3464] 0.00 : ffff800010087d24: msr apiakeylo_el1, x0 0.00 : ffff800010087d28: ldr x0, [x19, #3472] 0.00 : ffff800010087d2c: msr apiakeyhi_el1, x0 : __ptrauth_key_install(APIB, keys->apib); 0.00 : ffff800010087d30: ldr x0, [x19, #3480] 0.00 : ffff800010087d34: msr apibkeylo_el1, x0 0.00 : ffff800010087d38: ldr x0, [x19, #3488] 0.00 : ffff800010087d3c: msr apibkeyhi_el1, x0 : __ptrauth_key_install(APDA, keys->apda); 0.00 : ffff800010087d40: ldr x0, [x19, #3496] 0.00 : ffff800010087d44: msr apdakeylo_el1, x0 0.00 : ffff800010087d48: ldr x0, [x19, #3504] 0.00 : ffff800010087d4c: msr apdakeyhi_el1, x0 : __ptrauth_key_install(APDB, keys->apdb); 0.00 : ffff800010087d50: ldr x0, [x19, #3512] 0.00 : ffff800010087d54: msr apdbkeylo_el1, x0 0.00 : ffff800010087d58: ldr x0, [x19, #3520] 0.00 : ffff800010087d5c: msr apdbkeyhi_el1, x0 0.00 : ffff800010087d60: b ffff800010087cd8 <__switch_to+0x98> : test_bit(): 0.00 : ffff800010087d64: adrp x0, ffff800011af2000 0.00 : ffff800010087d68: ldr x0, [x0, #1320] : system_supports_generic_auth(): : cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH_IMP_DEF)); : } : : static inline bool system_supports_generic_auth(void) : { : return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) && 0.00 : ffff800010087d6c: tbz x0, #41, ffff800010087d80 <__switch_to+0x140> : ptrauth_keys_switch(): : } : : if (system_supports_generic_auth()) : __ptrauth_key_install(APGA, keys->apga); 0.00 : ffff800010087d70: ldr x0, [x19, #3528] 0.00 : ffff800010087d74: msr apgakeylo_el1, x0 0.00 : ffff800010087d78: ldr x0, [x19, #3536] 0.00 : ffff800010087d7c: msr apgakeyhi_el1, x0 : ssbs_thread_switch(): : if (unlikely(next->flags & PF_KTHREAD)) 0.00 : ffff800010087d80: ldr w0, [x19, #44] : task_stack_page(): 0.00 : ffff800010087d84: ldr x21, [x19, #32] : ssbs_thread_switch(): 0.00 : ffff800010087d88: tbnz w0, #21, ffff800010087cf4 <__switch_to+0xb4> : if (cpu_have_feature(cpu_feature(SSBS))) 0.00 : ffff800010087d8c: mov w0, #0x1c // #28 0.00 : ffff800010087d90: bl ffff800010093ae8 0.00 : ffff800010087d94: tst w0, #0xff 0.00 : ffff800010087d98: b.ne ffff800010087cf4 <__switch_to+0xb4> // b.any : arm64_get_ssbd_state(): : : static inline int arm64_get_ssbd_state(void) : { : #ifdef CONFIG_ARM64_SSBD : extern int ssbd_state; : return ssbd_state; 0.00 : ffff800010087d9c: adrp x0, ffff800011909000 : ssbs_thread_switch(): : if ((arm64_get_ssbd_state() == ARM64_SSBD_FORCE_ENABLE) || 0.00 : ffff800010087da0: ldr w0, [x0, #2320] 0.00 : ffff800010087da4: cmp w0, #0x2 0.00 : ffff800010087da8: b.eq ffff800010087cf4 <__switch_to+0xb4> // b.none : test_bit(): 0.00 : ffff800010087dac: ldr x0, [x19] : ssbs_thread_switch(): 0.00 : ffff800010087db0: tst w0, #0x2000000 0.00 : ffff800010087db4: b.ne ffff800010087cf4 <__switch_to+0xb4> // b.any : if (compat_user_mode(regs)) 0.00 : ffff800010087db8: ldr x0, [x21, #16328] 0.00 : ffff800010087dbc: and x1, x0, #0x1f 0.00 : ffff800010087dc0: cmp x1, #0x10 0.00 : ffff800010087dc4: b.eq ffff800010087e28 <__switch_to+0x1e8> // b.none : else if (user_mode(regs)) 0.12 : ffff800010087dc8: tst x0, #0xf 0.00 : ffff800010087dcc: b.ne ffff800010087cf4 <__switch_to+0xb4> // b.any : set_ssbs_bit(): : regs->pmr_save = GIC_PRIO_IRQON; : } : : static inline void set_ssbs_bit(struct pt_regs *regs) : { : regs->pstate |= PSR_SSBS_BIT; 0.00 : ffff800010087dd0: orr x0, x0, #0x1000 0.06 : ffff800010087dd4: str x0, [x21, #16328] 0.00 : ffff800010087dd8: b ffff800010087cf4 <__switch_to+0xb4> : test_bit(): 0.00 : ffff800010087ddc: adrp x0, ffff800011af2000 0.00 : ffff800010087de0: ldr x0, [x0, #1320] : system_supports_generic_auth(): : return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) && 0.00 : ffff800010087de4: tbnz x0, #40, ffff800010087d70 <__switch_to+0x130> : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff800010087de8: b ffff800010087d64 <__switch_to+0x124> 0.00 : ffff800010087dec: b ffff800010087ce4 <__switch_to+0xa4> : test_bit(): 0.00 : ffff800010087df0: adrp x0, ffff800011af2000 0.00 : ffff800010087df4: ldr x0, [x0, #1320] : system_supports_address_auth(): : return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) && 0.00 : ffff800010087df8: tbnz x0, #38, ffff800010087d20 <__switch_to+0xe0> : arch_static_branch_jump(): 0.00 : ffff800010087dfc: b ffff800010087d14 <__switch_to+0xd4> 0.00 : ffff800010087e00: b ffff800010087cd4 <__switch_to+0x94> : arm64_kernel_unmapped_at_el0(): : : extern bool arm64_use_ng_mappings; : : static inline bool arm64_kernel_unmapped_at_el0(void) : { : return arm64_use_ng_mappings; 1.53 : ffff800010087e04: adrp x0, ffff800011af2000 : tls_thread_switch(): : else if (!arm64_kernel_unmapped_at_el0()) 0.00 : ffff800010087e08: ldrb w0, [x0, #2168] 0.00 : ffff800010087e0c: cbnz w0, ffff800010087c94 <__switch_to+0x54> : write_sysreg(0, tpidrro_el0); 0.00 : ffff800010087e10: msr tpidrro_el0, xzr : test_bit(): 0.00 : ffff800010087e14: ldr x0, [x19] : tls_thread_switch(): : write_sysreg(*task_user_tls(next), tpidr_el0); 0.00 : ffff800010087e18: tst w0, #0x400000 0.00 : ffff800010087e1c: b.eq ffff800010087ca0 <__switch_to+0x60> // b.none 0.00 : ffff800010087e20: ldr x0, [x19, #2616] 0.00 : ffff800010087e24: b ffff800010087ca4 <__switch_to+0x64> : set_compat_ssbs_bit(): : } : : static inline void set_compat_ssbs_bit(struct pt_regs *regs) : { : regs->pstate |= PSR_AA32_SSBS_BIT; 0.00 : ffff800010087e28: orr x0, x0, #0x800000 0.00 : ffff800010087e2c: str x0, [x21, #16328] 0.00 : ffff800010087e30: b ffff800010087cf4 <__switch_to+0xb4> Percent | Source code & Disassembly of vmlinux for cycles (1259 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001014dda0 : : irq_thread_fn(): : * preemtible - many of them need to sleep and wait for slow busses to : * complete. : */ : static irqreturn_t irq_thread_fn(struct irq_desc *desc, : struct irqaction *action) : { 2.70 : ffff80001014dda0: stp x29, x30, [sp, #-48]! 0.00 : ffff80001014dda4: mov x29, sp 11.41 : ffff80001014dda8: stp x19, x20, [sp, #16] 0.00 : ffff80001014ddac: mov x19, x1 3.64 : ffff80001014ddb0: str x21, [sp, #32] 0.00 : ffff80001014ddb4: mov x20, x0 : irqreturn_t ret; : : ret = action->thread_fn(action->irq, action->dev_id); 1.97 : ffff80001014ddb8: ldr x2, [x19, #32] 1.28 : ffff80001014ddbc: ldr w0, [x1, #56] 2.03 : ffff80001014ddc0: ldr x1, [x1, #8] 0.00 : ffff80001014ddc4: blr x2 0.99 : ffff80001014ddc8: mov w21, w0 : if (ret == IRQ_HANDLED) 0.00 : ffff80001014ddcc: cmp w0, #0x1 0.00 : ffff80001014ddd0: b.eq ffff80001014de10 // b.none : irq_finalize_oneshot(): : if (!(desc->istate & IRQS_ONESHOT) || 0.14 : ffff80001014ddd4: ldr w0, [x20, #172] 0.00 : ffff80001014ddd8: tbz w0, #5, ffff80001014ddfc 0.00 : ffff80001014dddc: ldr x1, [x19] 0.00 : ffff80001014dde0: adrp x0, ffff80001014c000 0.00 : ffff80001014dde4: add x0, x0, #0xea8 0.00 : ffff80001014dde8: cmp x1, x0 0.00 : ffff80001014ddec: b.eq ffff80001014ddfc // b.none 0.00 : ffff80001014ddf0: mov x1, x19 0.00 : ffff80001014ddf4: mov x0, x20 0.00 : ffff80001014ddf8: bl ffff80001014d228 : irq_thread_fn(): : atomic_inc(&desc->threads_handled); : : irq_finalize_oneshot(desc, action); : return ret; : } 28.19 : ffff80001014ddfc: mov w0, w21 1.66 : ffff80001014de00: ldr x21, [sp, #32] 1.33 : ffff80001014de04: ldp x19, x20, [sp, #16] 0.08 : ffff80001014de08: ldp x29, x30, [sp], #48 0.08 : ffff80001014de0c: ret : atomic_inc(&desc->threads_handled); 2.00 : ffff80001014de10: add x1, x20, #0xcc : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001014de14: b ffff80001014de28 0.41 : ffff80001014de18: b ffff80001014de28 : __lse_atomic_add(): : } : : ATOMIC_OP(andnot, stclr) : ATOMIC_OP(or, stset) : ATOMIC_OP(xor, steor) : ATOMIC_OP(add, stadd) 0.24 : ffff80001014de1c: mov w0, #0x1 // #1 0.39 : ffff80001014de20: stadd w0, [x1] 41.48 : ffff80001014de24: b ffff80001014ddd4 : __ll_sc_atomic_add(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff80001014de28: add x2, x20, #0xcc 0.00 : ffff80001014de2c: b ffff800010150080 <__irq_get_irqchip_state+0x120> 0.00 : ffff80001014de30: b ffff80001014ddd4 Percent | Source code & Disassembly of vmlinux for cycles (1232 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc3e60 : : radix_tree_lookup(): : * must manage lifetimes of leaf nodes (eg. RCU may also be used to free : * them safely). No RCU barriers are required to access or modify the : * returned item, however. : */ : void *radix_tree_lookup(const struct radix_tree_root *root, unsigned long index) : { 25.01 : ffff800010cc3e60: stp x29, x30, [sp, #-16]! : return __radix_tree_lookup(root, index, NULL, NULL); 0.00 : ffff800010cc3e64: mov x3, #0x0 // #0 0.00 : ffff800010cc3e68: mov x2, #0x0 // #0 : { 0.00 : ffff800010cc3e6c: mov x29, sp : return __radix_tree_lookup(root, index, NULL, NULL); 16.80 : ffff800010cc3e70: bl ffff800010cc3d78 <__radix_tree_lookup> : } 58.19 : ffff800010cc3e74: ldp x29, x30, [sp], #16 0.00 : ffff800010cc3e78: ret Percent | Source code & Disassembly of vmlinux for cycles (1209 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017bdf8 : : tick_nohz_idle_enter(): : * tick_nohz_idle_enter - prepare for entering idle on the current CPU : * : * Called when we start the idle loop. : */ : void tick_nohz_idle_enter(void) : { 1.88 : ffff80001017bdf8: stp x29, x30, [sp, #-32]! : arch_local_irq_disable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff80001017bdfc: mov x0, #0x60 // #96 : tick_nohz_idle_enter(): 1.55 : ffff80001017be00: mov x29, sp 0.00 : ffff80001017be04: str x19, [sp, #16] : arch_local_irq_disable(): 1.72 : ffff80001017be08: msr daifset, #0x2 : tick_nohz_idle_enter(): : : lockdep_assert_irqs_enabled(); : : local_irq_disable(); : : ts = this_cpu_ptr(&tick_cpu_sched); 11.46 : ffff80001017be0c: adrp x19, ffff800011525000 0.00 : ffff80001017be10: add x19, x19, #0xed0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.91 : ffff80001017be14: mrs x0, tpidr_el1 : tick_nohz_idle_enter(): 0.00 : ffff80001017be18: add x19, x19, x0 : : WARN_ON_ONCE(ts->timer_expires_base); 0.41 : ffff80001017be1c: ldr x0, [x19, #176] 0.00 : ffff80001017be20: cbnz x0, ffff80001017be6c : : ts->inidle = 1; 15.54 : ffff80001017be24: ldrb w0, [x19, #76] 0.00 : ffff80001017be28: orr w0, w0, #0x1 0.59 : ffff80001017be2c: strb w0, [x19, #76] : tick_nohz_start_idle(): : ts->idle_entrytime = ktime_get(); 0.00 : ffff80001017be30: bl ffff80001016c938 0.00 : ffff80001017be34: str x0, [x19, #120] : ts->idle_active = 1; 0.00 : ffff80001017be38: ldrb w1, [x19, #76] : arch_local_irq_enable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001017be3c: mov x0, #0xe0 // #224 : tick_nohz_start_idle(): 0.00 : ffff80001017be40: orr w1, w1, #0x4 0.00 : ffff80001017be44: strb w1, [x19, #76] : arch_local_irq_enable(): 0.50 : ffff80001017be48: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 9.61 : ffff80001017be4c: nop : tick_nohz_idle_enter(): : tick_nohz_start_idle(ts); : : local_irq_enable(); : } 0.00 : ffff80001017be50: ldr x19, [sp, #16] 0.00 : ffff80001017be54: ldp x29, x30, [sp], #32 0.00 : ffff80001017be58: ret : arch_local_irq_enable(): : pmr_sync(); 0.58 : ffff80001017be5c: dsb sy : tick_nohz_idle_enter(): 46.49 : ffff80001017be60: ldr x19, [sp, #16] 8.76 : ffff80001017be64: ldp x29, x30, [sp], #32 0.00 : ffff80001017be68: ret : WARN_ON_ONCE(ts->timer_expires_base); 0.00 : ffff80001017be6c: brk #0x800 0.00 : ffff80001017be70: b ffff80001017be24 Percent | Source code & Disassembly of vmlinux for cycles (1194 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017c040 : : tick_nohz_idle_exit(): : * Restart the idle tick when the CPU is woken up from idle : * This also exit the RCU extended quiescent state. The CPU : * can use RCU again after this function is called. : */ : void tick_nohz_idle_exit(void) : { 0.09 : ffff80001017c040: stp x29, x30, [sp, #-48]! : arch_local_irq_disable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff80001017c044: mov x0, #0x60 // #96 : tick_nohz_idle_exit(): 0.00 : ffff80001017c048: mov x29, sp 0.08 : ffff80001017c04c: stp x19, x20, [sp, #16] : struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 0.00 : ffff80001017c050: adrp x19, ffff800011525000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.09 : ffff80001017c054: mrs x1, tpidr_el1 : tick_nohz_idle_exit(): 0.00 : ffff80001017c058: add x19, x19, #0xed0 : { 0.00 : ffff80001017c05c: str x21, [sp, #32] : struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 0.00 : ffff80001017c060: add x19, x19, x1 : arch_local_irq_disable(): 0.00 : ffff80001017c064: msr daifset, #0x2 : tick_nohz_idle_exit(): : bool idle_active, tick_stopped; : ktime_t now; : : local_irq_disable(); : : WARN_ON_ONCE(!ts->inidle); 11.51 : ffff80001017c068: ldrb w0, [x19, #76] 0.00 : ffff80001017c06c: tbz w0, #0, ffff80001017c100 : WARN_ON_ONCE(ts->timer_expires_base); 6.85 : ffff80001017c070: ldr x0, [x19, #176] 0.00 : ffff80001017c074: cbnz x0, ffff80001017c0f8 : : ts->inidle = 0; 0.66 : ffff80001017c078: ldrb w0, [x19, #76] 0.00 : ffff80001017c07c: and w0, w0, #0xfffffffe 0.00 : ffff80001017c080: strb w0, [x19, #76] : idle_active = ts->idle_active; 0.00 : ffff80001017c084: ubfx x21, x0, #2, #1 : tick_stopped = ts->tick_stopped; 0.00 : ffff80001017c088: ubfx x20, x0, #1, #1 : : if (idle_active || tick_stopped) 0.00 : ffff80001017c08c: orr w0, w21, w20 0.00 : ffff80001017c090: cbnz w0, ffff80001017c0c4 : arch_local_irq_enable(): : asm volatile(ALTERNATIVE( 5.26 : ffff80001017c094: mov x0, #0xe0 // #224 0.17 : ffff80001017c098: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 13.08 : ffff80001017c09c: nop : tick_nohz_idle_exit(): : : if (tick_stopped) : __tick_nohz_idle_restart_tick(ts, now); : : local_irq_enable(); : } 0.00 : ffff80001017c0a0: ldp x19, x20, [sp, #16] 0.00 : ffff80001017c0a4: ldr x21, [sp, #32] 0.00 : ffff80001017c0a8: ldp x29, x30, [sp], #48 0.00 : ffff80001017c0ac: ret : arch_local_irq_enable(): : pmr_sync(); 0.93 : ffff80001017c0b0: dsb sy : tick_nohz_idle_exit(): 47.47 : ffff80001017c0b4: ldp x19, x20, [sp, #16] 12.33 : ffff80001017c0b8: ldr x21, [sp, #32] 0.00 : ffff80001017c0bc: ldp x29, x30, [sp], #48 0.00 : ffff80001017c0c0: ret 0.00 : ffff80001017c0c4: str x22, [x29, #40] : now = ktime_get(); 1.51 : ffff80001017c0c8: bl ffff80001016c938 0.00 : ffff80001017c0cc: mov x22, x0 : if (idle_active) 0.00 : ffff80001017c0d0: cbz w21, ffff80001017c0e4 : tick_nohz_stop_idle(ts, now); 0.00 : ffff80001017c0d4: mov x1, x0 0.00 : ffff80001017c0d8: mov x0, x19 0.00 : ffff80001017c0dc: bl ffff80001017b6c0 : if (tick_stopped) 0.00 : ffff80001017c0e0: cbz w20, ffff80001017c108 : __tick_nohz_idle_restart_tick(ts, now); 0.00 : ffff80001017c0e4: mov x1, x22 0.00 : ffff80001017c0e8: mov x0, x19 0.00 : ffff80001017c0ec: bl ffff80001017b590 <__tick_nohz_idle_restart_tick> 0.00 : ffff80001017c0f0: ldr x22, [x29, #40] 0.00 : ffff80001017c0f4: b ffff80001017c094 : WARN_ON_ONCE(ts->timer_expires_base); 0.00 : ffff80001017c0f8: brk #0x800 0.00 : ffff80001017c0fc: b ffff80001017c078 : WARN_ON_ONCE(!ts->inidle); 0.00 : ffff80001017c100: brk #0x800 0.00 : ffff80001017c104: b ffff80001017c070 0.00 : ffff80001017c108: ldr x22, [x29, #40] 0.00 : ffff80001017c10c: b ffff80001017c094 Percent | Source code & Disassembly of vmlinux for cycles (1180 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012aab8 : : update_curr_rt(): : /* : * Update the current task's runtime statistics. Skip current tasks that : * are not in our scheduling class. : */ : static void update_curr_rt(struct rq *rq) : { 0.79 : ffff80001012aab8: stp x29, x30, [sp, #-64]! : struct task_struct *curr = rq->curr; : struct sched_rt_entity *rt_se = &curr->rt; : u64 delta_exec; : u64 now; : : if (curr->sched_class != &rt_sched_class) 0.00 : ffff80001012aabc: adrp x1, ffff800010d06000 0.17 : ffff80001012aac0: add x1, x1, #0x1f8 : { 0.00 : ffff80001012aac4: mov x29, sp 1.37 : ffff80001012aac8: str x19, [sp, #16] : struct task_struct *curr = rq->curr; 1.35 : ffff80001012aacc: ldr x19, [x0, #2352] : if (curr->sched_class != &rt_sched_class) 1.31 : ffff80001012aad0: ldr x2, [x19, #128] 0.00 : ffff80001012aad4: cmp x2, x1 0.00 : ffff80001012aad8: b.eq ffff80001012aae8 // b.none : if (sched_rt_runtime_exceeded(rt_rq)) : resched_curr(rq); : raw_spin_unlock(&rt_rq->rt_runtime_lock); : } : } : } 1.10 : ffff80001012aadc: ldr x19, [sp, #16] 0.53 : ffff80001012aae0: ldp x29, x30, [sp], #64 0.00 : ffff80001012aae4: ret 2.48 : ffff80001012aae8: str x21, [x29, #32] : now = rq_clock_task(rq); 0.84 : ffff80001012aaec: ldr x3, [x0, #2432] : delta_exec = now - curr->se.exec_start; 0.26 : ffff80001012aaf0: ldr x21, [x19, #264] 0.00 : ffff80001012aaf4: sub x21, x3, x21 : if (unlikely((s64)delta_exec <= 0)) 0.00 : ffff80001012aaf8: cmp x21, #0x0 0.00 : ffff80001012aafc: b.le ffff80001012ac94 0.87 : ffff80001012ab00: str x20, [x29, #24] : curr->se.sum_exec_runtime += delta_exec; 1.89 : ffff80001012ab04: ldr x2, [x19, #272] : get_running_cputimer(): : */ : #ifdef CONFIG_POSIX_TIMERS : static inline : struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk) : { : struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 0.33 : ffff80001012ab08: ldr x1, [x19, #1640] : update_curr_rt(): 0.00 : ffff80001012ab0c: add x2, x2, x21 0.44 : ffff80001012ab10: str x2, [x19, #272] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.08 : ffff80001012ab14: ldr w2, [x1, #344] : get_running_cputimer(): : : /* : * Check whether posix CPU timers are active. If not the thread : * group accounting is not active either. Lockless check. : */ : if (!READ_ONCE(tsk->signal->posix_cputimers.timers_active)) 0.00 : ffff80001012ab18: cbz w2, ffff80001012ab48 : account_group_exec_runtime(): : static inline void account_group_exec_runtime(struct task_struct *tsk, : unsigned long long ns) : { : struct thread_group_cputimer *cputimer = get_running_cputimer(tsk); : : if (!cputimer) 0.00 : ffff80001012ab1c: ldr x4, [x19, #1648] : get_running_cputimer(): : struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 0.00 : ffff80001012ab20: add x2, x1, #0xf8 : account_group_exec_runtime(): : if (!cputimer) 0.00 : ffff80001012ab24: cmp x4, #0x0 0.00 : ffff80001012ab28: ccmp x2, #0x0, #0x4, ne // ne = any 0.00 : ffff80001012ab2c: b.eq ffff80001012ab48 // b.none : return; : : atomic64_add(ns, &cputimer->cputime_atomic.sum_exec_runtime); 0.00 : ffff80001012ab30: add x2, x1, #0x108 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001012ab34: b ffff80001012ac20 0.00 : ffff80001012ab38: b ffff80001012ac20 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff80001012ab3c: mov x4, x21 0.00 : ffff80001012ab40: stadd x4, [x2] 0.00 : ffff80001012ab44: nop : cgroup_account_cputime(): : static inline void cgroup_account_cputime(struct task_struct *task, : u64 delta_exec) : { : struct cgroup *cgrp; : : cpuacct_charge(task, delta_exec); 7.38 : ffff80001012ab48: mov x1, x21 0.00 : ffff80001012ab4c: mov x20, x0 0.00 : ffff80001012ab50: mov x0, x19 : update_curr_rt(): : curr->se.exec_start = now; 0.41 : ffff80001012ab54: str x3, [x19, #264] : cgroup_account_cputime(): 0.00 : ffff80001012ab58: bl ffff800010136480 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.09 : ffff80001012ab5c: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.17 : ffff80001012ab60: ldr x0, [x19, #2000] : task_dfl_cgroup(): : return task_css_set(task)->dfl_cgrp; 0.17 : ffff80001012ab64: ldr x0, [x0, #88] : cgroup_parent(): : if (parent_css) 2.53 : ffff80001012ab68: ldr x1, [x0, #232] 0.00 : ffff80001012ab6c: cbz x1, ffff80001012ab78 : cgroup_account_cputime(): : : rcu_read_lock(); : cgrp = task_dfl_cgroup(task); : if (cgroup_parent(cgrp)) : __cgroup_account_cputime(cgrp, delta_exec); 0.00 : ffff80001012ab70: mov x1, x21 0.00 : ffff80001012ab74: bl ffff8000101973e8 <__cgroup_account_cputime> : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 2.56 : ffff80001012ab78: bl ffff800010160d20 <__rcu_read_unlock> : rt_bandwidth_enabled(): : #endif /* CONFIG_FAIR_GROUP_SCHED */ : }; : : static inline int rt_bandwidth_enabled(void) : { : return sysctl_sched_rt_runtime >= 0; 0.00 : ffff80001012ab7c: adrp x1, ffff800011920000 : update_curr_rt(): : struct sched_rt_entity *rt_se = &curr->rt; 0.68 : ffff80001012ab80: add x0, x19, #0x1c0 : for_each_sched_rt_entity(rt_se) { 0.00 : ffff80001012ab84: ldr w1, [x1, #912] 0.00 : ffff80001012ab88: cmp w1, #0x0 0.00 : ffff80001012ab8c: ccmp x0, #0x0, #0x4, ge // ge = tcont 0.00 : ffff80001012ab90: b.ne ffff80001012aba4 // b.any 0.00 : ffff80001012ab94: ldp x20, x21, [x29, #24] : } 0.00 : ffff80001012ab98: ldr x19, [sp, #16] 0.00 : ffff80001012ab9c: ldp x29, x30, [sp], #64 0.00 : ffff80001012aba0: ret : __read_once_size(): 5.27 : ffff80001012aba4: ldr w2, [x19, #68] : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012aba8: adrp x1, ffff800011909000 0.00 : ffff80001012abac: add x1, x1, #0x928 0.00 : ffff80001012abb0: adrp x0, ffff800011528000 1.44 : ffff80001012abb4: add x0, x0, #0xe80 0.09 : ffff80001012abb8: ldr x19, [x1, w2, uxtw #3] 0.00 : ffff80001012abbc: add x19, x0, x19 : update_curr_rt(): : if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { 0.75 : ffff80001012abc0: ldr x0, [x19, #2200] 0.00 : ffff80001012abc4: cmn x0, #0x1 0.00 : ffff80001012abc8: b.eq ffff80001012ab94 // b.none 3.15 : ffff80001012abcc: stp x22, x23, [x29, #40] : raw_spin_lock(&rt_rq->rt_runtime_lock); 0.00 : ffff80001012abd0: add x22, x19, #0x200 0.00 : ffff80001012abd4: add x23, x19, #0x8a0 0.00 : ffff80001012abd8: mov x0, x23 0.25 : ffff80001012abdc: bl ffff800010cd80f0 <_raw_spin_lock> : rt_rq->rt_time += delta_exec; 6.83 : ffff80001012abe0: ldr x0, [x22, #1680] : sched_rt_runtime_exceeded(): : if (rt_rq->rt_throttled) 9.33 : ffff80001012abe4: ldr w1, [x22, #1676] : update_curr_rt(): : rt_rq->rt_time += delta_exec; 0.00 : ffff80001012abe8: add x21, x21, x0 0.00 : ffff80001012abec: str x21, [x22, #1680] : sched_rt_runtime_exceeded(): : if (rt_rq->rt_throttled) 0.00 : ffff80001012abf0: cbnz w1, ffff80001012ac78 0.34 : ffff80001012abf4: str x24, [x29, #56] : sched_rt_period(): : return ktime_to_ns(def_rt_bandwidth.rt_period); 0.00 : ffff80001012abf8: adrp x24, ffff800011afc000 0.00 : ffff80001012abfc: add x24, x24, #0x0 : sched_rt_runtime_exceeded(): : u64 runtime = sched_rt_runtime(rt_rq); 0.27 : ffff80001012ac00: ldr x0, [x19, #2200] : if (runtime >= sched_rt_period(rt_rq)) 0.17 : ffff80001012ac04: ldr x1, [x24, #8] 0.00 : ffff80001012ac08: cmp x0, x1 0.00 : ffff80001012ac0c: b.cs ffff80001012ac18 // b.hs, b.nlast : balance_runtime(): : if (rt_rq->rt_time > rt_rq->rt_runtime) { 0.44 : ffff80001012ac10: cmp x21, x0 0.00 : ffff80001012ac14: b.hi ffff80001012ac2c // b.pmore 41.56 : ffff80001012ac18: ldr x24, [x29, #56] 0.00 : ffff80001012ac1c: b ffff80001012ac80 : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff80001012ac20: add x1, x1, #0x108 0.00 : ffff80001012ac24: b ffff80001012b4cc 0.00 : ffff80001012ac28: b ffff80001012ab48 : balance_runtime(): 0.00 : ffff80001012ac2c: mov x0, x22 0.00 : ffff80001012ac30: bl ffff800010128f90 : sched_rt_runtime_exceeded(): : runtime = sched_rt_runtime(rt_rq); 0.00 : ffff80001012ac34: ldr x0, [x19, #2200] : if (runtime == RUNTIME_INF) 0.00 : ffff80001012ac38: cmn x0, #0x1 0.00 : ffff80001012ac3c: b.eq ffff80001012ac18 // b.none : if (rt_rq->rt_time > runtime) { 0.00 : ffff80001012ac40: ldr x1, [x22, #1680] 0.00 : ffff80001012ac44: cmp x0, x1 0.00 : ffff80001012ac48: b.cs ffff80001012ac18 // b.hs, b.nlast : if (likely(rt_b->rt_runtime)) { 0.00 : ffff80001012ac4c: ldr x0, [x24, #16] 0.00 : ffff80001012ac50: cbz x0, ffff80001012ac9c : printk_deferred_once("sched: RT throttling activated\n"); 0.00 : ffff80001012ac54: adrp x0, ffff800011ad8000 : rt_rq->rt_throttled = 1; 0.00 : ffff80001012ac58: mov w1, #0x1 // #1 0.00 : ffff80001012ac5c: str w1, [x22, #1676] : printk_deferred_once("sched: RT throttling activated\n"); 0.00 : ffff80001012ac60: ldrb w2, [x0, #1209] 0.00 : ffff80001012ac64: cbz w2, ffff80001012acb0 : sched_rt_rq_dequeue(): : dequeue_top_rt_rq(rt_rq); 0.00 : ffff80001012ac68: mov x0, x22 0.00 : ffff80001012ac6c: bl ffff800010129378 0.00 : ffff80001012ac70: ldr x24, [x29, #56] 0.00 : ffff80001012ac74: nop : update_curr_rt(): : resched_curr(rq); 0.00 : ffff80001012ac78: mov x0, x20 0.00 : ffff80001012ac7c: bl ffff800010114bb8 : raw_spin_unlock(&rt_rq->rt_runtime_lock); 0.90 : ffff80001012ac80: mov x0, x23 0.00 : ffff80001012ac84: bl ffff800010cd7c38 <_raw_spin_unlock> 0.17 : ffff80001012ac88: ldp x20, x21, [x29, #24] 1.14 : ffff80001012ac8c: ldp x22, x23, [x29, #40] 0.00 : ffff80001012ac90: b ffff80001012aadc 0.08 : ffff80001012ac94: ldr x21, [x29, #32] 0.00 : ffff80001012ac98: b ffff80001012aadc : sched_rt_runtime_exceeded(): : rt_rq->rt_time = 0; 0.00 : ffff80001012ac9c: str xzr, [x22, #1680] 0.00 : ffff80001012aca0: ldr w0, [x19, #2188] : if (rt_rq_throttled(rt_rq)) { 0.00 : ffff80001012aca4: cbnz w0, ffff80001012ac68 0.00 : ffff80001012aca8: ldr x24, [x29, #56] 0.00 : ffff80001012acac: b ffff80001012ac80 : printk_deferred_once("sched: RT throttling activated\n"); 0.00 : ffff80001012acb0: strb w1, [x0, #1209] 0.00 : ffff80001012acb4: adrp x0, ffff8000111c4000 0.00 : ffff80001012acb8: add x0, x0, #0xe0 0.00 : ffff80001012acbc: bl ffff80001014a96c 0.00 : ffff80001012acc0: ldr w0, [x19, #2188] : if (rt_rq_throttled(rt_rq)) { 0.00 : ffff80001012acc4: cbz w0, ffff80001012aca8 0.00 : ffff80001012acc8: b ffff80001012ac68 Percent | Source code & Disassembly of vmlinux for cycles (1232 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc4ca0 : : __rb_insert(): : : static __always_inline void : __rb_insert(struct rb_node *node, struct rb_root *root, : void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) : { : struct rb_node *parent = rb_red_parent(node), *gparent, *tmp; 66.44 : ffff800010cc4ca0: ldr x3, [x0] : : while (true) { : /* : * Loop invariant: node is red. : */ : if (unlikely(!parent)) { 0.00 : ffff800010cc4ca4: cbz x3, ffff800010cc4df8 : * If there is a black parent, we are done. : * Otherwise, take some corrective action as, : * per 4), we don't want a red root or two : * consecutive red nodes. : */ : if(rb_is_black(parent)) 1.39 : ffff800010cc4ca8: ldr x2, [x3] 0.00 : ffff800010cc4cac: tbnz w2, #0, ffff800010cc4d14 : break; : : gparent = rb_red_parent(parent); : : tmp = gparent->rb_right; 0.00 : ffff800010cc4cb0: ldr x4, [x2, #8] : rb_red_parent(): : return (struct rb_node *)red->__rb_parent_color; 0.00 : ffff800010cc4cb4: mov x6, x2 : __rb_insert(): : if (parent != tmp) { /* parent == gparent->rb_left */ 0.00 : ffff800010cc4cb8: cmp x4, x3 0.00 : ffff800010cc4cbc: b.eq ffff800010cc4d18 // b.none : if (tmp && rb_is_red(tmp)) { 0.00 : ffff800010cc4cc0: cbz x4, ffff800010cc4ccc 0.00 : ffff800010cc4cc4: ldr x5, [x4] 0.00 : ffff800010cc4cc8: tbz w5, #0, ffff800010cc4d90 : parent = rb_parent(node); : rb_set_parent_color(node, parent, RB_RED); : continue; : } : : tmp = parent->rb_right; 0.00 : ffff800010cc4ccc: ldr x4, [x3, #8] 0.00 : ffff800010cc4cd0: mov x5, x3 : if (node == tmp) { 0.00 : ffff800010cc4cd4: cmp x4, x0 0.00 : ffff800010cc4cd8: b.eq ffff800010cc4dbc // b.none : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010cc4cdc: str x4, [x2, #16] 0.00 : ffff800010cc4ce0: str x2, [x3, #8] : __rb_insert(): : * / \ : * n U : */ : WRITE_ONCE(gparent->rb_left, tmp); /* == parent->rb_right */ : WRITE_ONCE(parent->rb_right, gparent); : if (tmp) 0.00 : ffff800010cc4ce4: cbz x4, ffff800010cc4cf0 : rb_set_parent_color(): : } : : static inline void rb_set_parent_color(struct rb_node *rb, : struct rb_node *p, int color) : { : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4ce8: orr x0, x2, #0x1 0.00 : ffff800010cc4cec: str x0, [x4] : __rb_rotate_set_parents(): : struct rb_node *parent = rb_parent(old); 0.00 : ffff800010cc4cf0: ldr x0, [x2] : new->__rb_parent_color = old->__rb_parent_color; 0.00 : ffff800010cc4cf4: str x0, [x3] : rb_set_parent_color(): 0.00 : ffff800010cc4cf8: str x5, [x2] : __rb_change_child(): : : static inline void : __rb_change_child(struct rb_node *old, struct rb_node *new, : struct rb_node *parent, struct rb_root *root) : { : if (parent) { 0.00 : ffff800010cc4cfc: ands x0, x0, #0xfffffffffffffffc 0.00 : ffff800010cc4d00: b.eq ffff800010cc4d88 // b.none : if (parent->rb_left == old) 0.00 : ffff800010cc4d04: ldr x1, [x0, #16] 0.00 : ffff800010cc4d08: cmp x2, x1 0.00 : ffff800010cc4d0c: b.eq ffff800010cc4e04 // b.none : __write_once_size(): 0.00 : ffff800010cc4d10: str x5, [x0, #8] : rb_insert_color(): : }; : : void rb_insert_color(struct rb_node *node, struct rb_root *root) : { : __rb_insert(node, root, dummy_rotate); : } 6.94 : ffff800010cc4d14: ret : __rb_insert(): : tmp = gparent->rb_left; 0.00 : ffff800010cc4d18: ldr x5, [x2, #16] : if (tmp && rb_is_red(tmp)) { 0.00 : ffff800010cc4d1c: cbz x5, ffff800010cc4d28 0.00 : ffff800010cc4d20: ldr x7, [x5] 0.00 : ffff800010cc4d24: tbz w7, #0, ffff800010cc4e0c : tmp = parent->rb_left; 0.00 : ffff800010cc4d28: ldr x3, [x4, #16] 0.00 : ffff800010cc4d2c: mov x5, x4 : if (node == tmp) { 0.00 : ffff800010cc4d30: cmp x3, x0 0.00 : ffff800010cc4d34: b.ne ffff800010cc4d60 // b.any : tmp = node->rb_right; 0.00 : ffff800010cc4d38: ldr x3, [x0, #8] : __write_once_size(): 0.00 : ffff800010cc4d3c: str x3, [x4, #16] 0.00 : ffff800010cc4d40: str x4, [x0, #8] : __rb_insert(): : if (tmp) 0.00 : ffff800010cc4d44: cbz x3, ffff800010cc4d50 : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4d48: orr x5, x4, #0x1 0.00 : ffff800010cc4d4c: str x5, [x3] 0.00 : ffff800010cc4d50: str x0, [x4] 0.00 : ffff800010cc4d54: mov x5, x0 : __rb_insert(): : tmp = node->rb_left; 0.00 : ffff800010cc4d58: mov x4, x0 0.00 : ffff800010cc4d5c: ldr x3, [x0, #16] : __write_once_size(): 0.00 : ffff800010cc4d60: str x3, [x2, #8] 0.00 : ffff800010cc4d64: str x2, [x4, #16] : __rb_insert(): : if (tmp) 0.00 : ffff800010cc4d68: cbz x3, ffff800010cc4d74 : rb_set_parent_color(): 0.00 : ffff800010cc4d6c: orr x0, x2, #0x1 0.00 : ffff800010cc4d70: str x0, [x3] : __rb_rotate_set_parents(): : struct rb_node *parent = rb_parent(old); 0.00 : ffff800010cc4d74: ldr x0, [x2] : new->__rb_parent_color = old->__rb_parent_color; 0.00 : ffff800010cc4d78: str x0, [x4] : rb_set_parent_color(): 0.00 : ffff800010cc4d7c: str x5, [x2] : __rb_change_child(): : if (parent) { 0.00 : ffff800010cc4d80: ands x0, x0, #0xfffffffffffffffc 0.00 : ffff800010cc4d84: b.ne ffff800010cc4d04 // b.any : __write_once_size(): 0.00 : ffff800010cc4d88: str x5, [x1] : rb_insert_color(): : } 0.00 : ffff800010cc4d8c: ret : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4d90: orr x0, x2, #0x1 0.00 : ffff800010cc4d94: str x0, [x4] 0.00 : ffff800010cc4d98: str x0, [x3] : __rb_insert(): : parent = rb_parent(node); 0.00 : ffff800010cc4d9c: ldr x3, [x2] 0.00 : ffff800010cc4da0: and x3, x3, #0xfffffffffffffffc : rb_set_parent_color(): 0.00 : ffff800010cc4da4: str x3, [x2] : __rb_insert(): : if (unlikely(!parent)) { 0.00 : ffff800010cc4da8: cbz x3, ffff800010cc4df4 : if(rb_is_black(parent)) 0.00 : ffff800010cc4dac: ldr x2, [x3] : rb_red_parent(): : return (struct rb_node *)red->__rb_parent_color; 0.00 : ffff800010cc4db0: mov x0, x6 : __rb_insert(): : if(rb_is_black(parent)) 0.00 : ffff800010cc4db4: tbz w2, #0, ffff800010cc4cb0 : rb_insert_color(): : } 0.00 : ffff800010cc4db8: ret : __rb_insert(): : tmp = node->rb_left; 0.00 : ffff800010cc4dbc: ldr x4, [x0, #16] : __write_once_size(): 0.00 : ffff800010cc4dc0: str x4, [x3, #8] 0.00 : ffff800010cc4dc4: str x3, [x0, #16] : __rb_insert(): : if (tmp) 0.00 : ffff800010cc4dc8: cbz x4, ffff800010cc4dd4 : rb_set_parent_color(): 0.00 : ffff800010cc4dcc: orr x5, x3, #0x1 0.00 : ffff800010cc4dd0: str x5, [x4] 0.00 : ffff800010cc4dd4: str x0, [x3] : __rb_insert(): : tmp = node->rb_right; 0.00 : ffff800010cc4dd8: mov x3, x0 : rb_set_parent_color(): 0.00 : ffff800010cc4ddc: mov x5, x0 : __rb_insert(): 0.00 : ffff800010cc4de0: ldr x4, [x0, #8] : __write_once_size(): 0.00 : ffff800010cc4de4: str x4, [x2, #16] 0.00 : ffff800010cc4de8: str x2, [x3, #8] : __rb_insert(): : if (tmp) 0.00 : ffff800010cc4dec: cbnz x4, ffff800010cc4ce8 0.00 : ffff800010cc4df0: b ffff800010cc4cf0 : rb_red_parent(): : return (struct rb_node *)red->__rb_parent_color; 0.00 : ffff800010cc4df4: mov x0, x2 : rb_set_parent_color(): 11.89 : ffff800010cc4df8: mov x1, #0x1 // #1 6.81 : ffff800010cc4dfc: str x1, [x0] : rb_insert_color(): : } 6.53 : ffff800010cc4e00: ret : __write_once_size(): 0.00 : ffff800010cc4e04: str x5, [x0, #16] : rb_insert_color(): 0.00 : ffff800010cc4e08: ret : rb_set_parent_color(): 0.00 : ffff800010cc4e0c: orr x0, x2, #0x1 0.00 : ffff800010cc4e10: str x0, [x5] 0.00 : ffff800010cc4e14: str x0, [x3] : __rb_insert(): : parent = rb_parent(node); 0.00 : ffff800010cc4e18: ldr x3, [x2] 0.00 : ffff800010cc4e1c: and x3, x3, #0xfffffffffffffffc : rb_set_parent_color(): 0.00 : ffff800010cc4e20: str x3, [x2] 0.00 : ffff800010cc4e24: b ffff800010cc4da8 Percent | Source code & Disassembly of vmlinux for cycles (1101 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001071e2e8 : : queue_poll(): : qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); : qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); : } : : static int queue_poll(struct arm_smmu_queue_poll *qp) : { 16.01 : ffff80001071e2e8: stp x29, x30, [sp, #-32]! 0.00 : ffff80001071e2ec: mov x29, sp 15.22 : ffff80001071e2f0: str x19, [sp, #16] 0.00 : ffff80001071e2f4: mov x19, x0 : if (ktime_compare(ktime_get(), qp->timeout) > 0) 0.00 : ffff80001071e2f8: bl ffff80001016c938 : ktime_compare(): : */ : static inline int ktime_compare(const ktime_t cmp1, const ktime_t cmp2) : { : if (cmp1 < cmp2) : return -1; : if (cmp1 > cmp2) 0.00 : ffff80001071e2fc: ldr x1, [x19] 0.00 : ffff80001071e300: cmp x0, x1 0.00 : ffff80001071e304: b.gt ffff80001071e370 : queue_poll(): : return -ETIMEDOUT; : : if (qp->wfe) { 0.00 : ffff80001071e308: ldrb w0, [x19, #16] 0.00 : ffff80001071e30c: cbnz w0, ffff80001071e35c : wfe(); : } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) { 34.22 : ffff80001071e310: ldr w0, [x19, #12] 0.00 : ffff80001071e314: add w0, w0, #0x1 0.10 : ffff80001071e318: str w0, [x19, #12] 0.00 : ffff80001071e31c: cmp w0, #0x9 0.00 : ffff80001071e320: b.ls ffff80001071e348 // b.plast : cpu_relax(); : } else { : udelay(qp->delay); 0.00 : ffff80001071e324: ldr w0, [x19, #8] 0.00 : ffff80001071e328: bl ffff800010cb7b80 <__udelay> : qp->delay *= 2; 0.00 : ffff80001071e32c: ldr w1, [x19, #8] : qp->spin_cnt = 0; 0.00 : ffff80001071e330: mov w0, #0x0 // #0 : qp->delay *= 2; 0.00 : ffff80001071e334: lsl w1, w1, #1 : qp->spin_cnt = 0; 0.00 : ffff80001071e338: stp w1, wzr, [x19, #8] : } : : return 0; : } 0.00 : ffff80001071e33c: ldr x19, [sp, #16] 0.00 : ffff80001071e340: ldp x29, x30, [sp], #32 0.00 : ffff80001071e344: ret : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff80001071e348: yield : queue_poll(): : return 0; 0.00 : ffff80001071e34c: mov w0, #0x0 // #0 : } 34.27 : ffff80001071e350: ldr x19, [sp, #16] 0.18 : ffff80001071e354: ldp x29, x30, [sp], #32 0.00 : ffff80001071e358: ret : wfe(); 0.00 : ffff80001071e35c: wfe : return 0; 0.00 : ffff80001071e360: mov w0, #0x0 // #0 : } 0.00 : ffff80001071e364: ldr x19, [sp, #16] 0.00 : ffff80001071e368: ldp x29, x30, [sp], #32 0.00 : ffff80001071e36c: ret : return -ETIMEDOUT; 0.00 : ffff80001071e370: mov w0, #0xffffff92 // #-110 0.00 : ffff80001071e374: b ffff80001071e33c Percent | Source code & Disassembly of vmlinux for cycles (1087 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011e278 : : select_idle_sibling(): : : /* : * Try and locate an idle core/thread in the LLC cache domain. : */ : static int select_idle_sibling(struct task_struct *p, int prev, int target) : { 0.00 : ffff80001011e278: stp x29, x30, [sp, #-96]! 0.00 : ffff80001011e27c: mov x29, sp 0.65 : ffff80001011e280: stp x19, x20, [sp, #16] 9.99 : ffff80001011e284: mov w20, w1 13.40 : ffff80001011e288: str x21, [sp, #32] 0.00 : ffff80001011e28c: mov w21, w2 0.00 : ffff80001011e290: str x27, [sp, #80] 0.09 : ffff80001011e294: mov x27, x0 : struct sched_domain *sd; : int i, recent_used_cpu; : : if (available_idle_cpu(target) || sched_idle_cpu(target)) 0.09 : ffff80001011e298: mov w0, w2 0.00 : ffff80001011e29c: bl ffff800010118870 28.83 : ffff80001011e2a0: cbnz w0, ffff80001011e498 0.00 : ffff80001011e2a4: stp x22, x23, [x29, #40] : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff80001011e2a8: adrp x23, ffff800011909000 0.00 : ffff80001011e2ac: stp x24, x25, [x29, #56] 0.00 : ffff80001011e2b0: sxtw x22, w21 0.00 : ffff80001011e2b4: add x25, x23, #0x928 0.00 : ffff80001011e2b8: adrp x24, ffff800011528000 0.00 : ffff80001011e2bc: add x19, x24, #0xe80 0.00 : ffff80001011e2c0: ldr x0, [x25, x22, lsl #3] 0.00 : ffff80001011e2c4: add x0, x19, x0 0.00 : ffff80001011e2c8: ldr w1, [x0, #4] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff80001011e2cc: ldr w0, [x0, #160] 0.00 : ffff80001011e2d0: cmp w1, #0x0 0.00 : ffff80001011e2d4: ccmp w0, w1, #0x0, ne // ne = any 0.00 : ffff80001011e2d8: b.eq ffff80001011e500 // b.none : select_idle_sibling(): : return target; : : /* : * If the previous CPU is cache affine and idle, don't be stupid: : */ : if (prev != target && cpus_share_cache(prev, target) && 0.00 : ffff80001011e2dc: cmp w21, w20 0.00 : ffff80001011e2e0: b.ne ffff80001011e4b4 // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001011e2e4: mrs x0, sp_el0 : is_per_cpu_kthread(): : #endif : : #ifdef CONFIG_SMP : static inline bool is_per_cpu_kthread(struct task_struct *p) : { : if (!(p->flags & PF_KTHREAD)) 0.00 : ffff80001011e2e8: ldr w1, [x0, #44] 0.00 : ffff80001011e2ec: tbz w1, #21, ffff80001011e2fc : return false; : : if (p->nr_cpus_allowed != 1) 0.00 : ffff80001011e2f0: ldr w0, [x0, #732] 0.00 : ffff80001011e2f4: cmp w0, #0x1 0.00 : ffff80001011e2f8: b.eq ffff80001011e55c // b.none : select_idle_sibling(): : this_rq()->nr_running <= 1) { : return prev; : } : : /* Check a recently used CPU as a potential idle candidate: */ : recent_used_cpu = p->recent_used_cpu; 0.00 : ffff80001011e2fc: ldr w19, [x27, #96] : if (recent_used_cpu != prev && 0.00 : ffff80001011e300: cmp w20, w19 0.00 : ffff80001011e304: ccmp w21, w19, #0x4, ne // ne = any 0.00 : ffff80001011e308: b.ne ffff80001011e58c // b.any : */ : p->recent_used_cpu = prev; : return recent_used_cpu; : } : : sd = rcu_dereference(per_cpu(sd_llc, target)); 0.00 : ffff80001011e30c: add x1, x23, #0x928 0.00 : ffff80001011e310: adrp x0, ffff80001151f000 0.00 : ffff80001011e314: add x0, x0, #0x118 0.00 : ffff80001011e318: ldr x1, [x1, x22, lsl #3] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011e31c: ldr x1, [x0, x1] : select_idle_sibling(): : if (!sd) 0.00 : ffff80001011e320: cbz x1, ffff80001011e500 0.00 : ffff80001011e324: str x26, [x29, #72] : select_idle_cpu(): : struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); 0.00 : ffff80001011e328: adrp x22, ffff80001151f000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011e32c: mrs x3, tpidr_el1 : __read_once_size(): 0.00 : ffff80001011e330: ldr x26, [x0, x3] : select_idle_cpu(): 0.00 : ffff80001011e334: add x22, x22, #0x58 0.00 : ffff80001011e338: add x22, x22, x3 : if (!this_sd) 0.00 : ffff80001011e33c: cbz x26, ffff80001011e634 : avg_idle = this_rq()->avg_idle / 512; 0.00 : ffff80001011e340: add x0, x24, #0xe80 0.00 : ffff80001011e344: str x28, [x29, #88] 0.00 : ffff80001011e348: add x0, x0, x3 : u64 span_avg = sd->span_weight * avg_idle; 0.00 : ffff80001011e34c: ldr w4, [x1, #128] : avg_cost = this_sd->avg_scan_cost + 1; 0.00 : ffff80001011e350: ldr x2, [x26, #96] 0.00 : ffff80001011e354: add x19, x1, #0x88 0.00 : ffff80001011e358: sub w28, w21, #0x1 : avg_idle = this_rq()->avg_idle / 512; 0.00 : ffff80001011e35c: ldr x20, [x0, #2824] : avg_cost = this_sd->avg_scan_cost + 1; 0.00 : ffff80001011e360: add x2, x2, #0x1 : avg_idle = this_rq()->avg_idle / 512; 0.00 : ffff80001011e364: lsr x20, x20, #9 : u64 span_avg = sd->span_weight * avg_idle; 0.00 : ffff80001011e368: mul x20, x20, x4 : if (span_avg > 4*avg_cost) 0.00 : ffff80001011e36c: cmp x20, x2, lsl #2 0.00 : ffff80001011e370: b.hi ffff80001011e50c // b.pmore : cpu_clock(): : { : } : : static inline u64 cpu_clock(int cpu) : { : return sched_clock(); 0.00 : ffff80001011e374: bl ffff80001017af20 0.00 : ffff80001011e378: mov x25, x0 : bitmap_and(): : static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, : const unsigned long *src2, unsigned int nbits) : { : if (small_const_nbits(nbits)) : return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0; : return __bitmap_and(dst, src1, src2, nbits); 0.00 : ffff80001011e37c: ldr x2, [x27, #736] 0.00 : ffff80001011e380: mov x1, x19 0.00 : ffff80001011e384: mov w3, #0x100 // #256 0.00 : ffff80001011e388: mov x0, x22 : select_idle_cpu(): : if (!--nr) 0.00 : ffff80001011e38c: mov w20, #0x3 // #3 : bitmap_and(): 0.00 : ffff80001011e390: bl ffff8000104850b8 <__bitmap_and> : select_idle_cpu(): : for_each_cpu_wrap(cpu, cpus, target) { 0.00 : ffff80001011e394: mov w3, #0x0 // #0 0.00 : ffff80001011e398: mov w2, w21 0.00 : ffff80001011e39c: mov x1, x22 0.00 : ffff80001011e3a0: mov w0, w28 0.00 : ffff80001011e3a4: bl ffff800010cb9368 0.00 : ffff80001011e3a8: mov w19, w0 0.00 : ffff80001011e3ac: cmp w0, #0xff 0.00 : ffff80001011e3b0: b.hi ffff80001011e460 // b.pmore : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff80001011e3b4: add x24, x24, #0xe80 0.00 : ffff80001011e3b8: add x23, x23, #0x928 0.00 : ffff80001011e3bc: b ffff80001011e3f8 0.00 : ffff80001011e3c0: ldr x5, [x23, w19, sxtw #3] : select_idle_cpu(): : for_each_cpu_wrap(cpu, cpus, target) { 0.00 : ffff80001011e3c4: mov w0, w19 : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff80001011e3c8: add x4, x4, x5 0.00 : ffff80001011e3cc: ldr w5, [x4, #4] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff80001011e3d0: ldr w4, [x4, #160] 0.00 : ffff80001011e3d4: cmp w5, #0x0 0.00 : ffff80001011e3d8: ccmp w4, w5, #0x0, ne // ne = any 0.00 : ffff80001011e3dc: b.eq ffff80001011e414 // b.none : select_idle_cpu(): : for_each_cpu_wrap(cpu, cpus, target) { 0.00 : ffff80001011e3e0: bl ffff800010cb9368 0.00 : ffff80001011e3e4: mov w19, w0 0.00 : ffff80001011e3e8: cmp w0, #0xff 0.00 : ffff80001011e3ec: b.hi ffff80001011e5e4 // b.pmore : if (!--nr) 0.00 : ffff80001011e3f0: subs w20, w20, #0x1 0.00 : ffff80001011e3f4: b.eq ffff80001011e488 // b.none : if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 0.00 : ffff80001011e3f8: mov w0, w19 0.00 : ffff80001011e3fc: bl ffff800010118870 : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff80001011e400: mov x4, x24 : select_idle_cpu(): : for_each_cpu_wrap(cpu, cpus, target) { 0.00 : ffff80001011e404: mov w3, #0x1 // #1 0.00 : ffff80001011e408: mov w2, w21 0.00 : ffff80001011e40c: mov x1, x22 : if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 0.00 : ffff80001011e410: cbz w0, ffff80001011e3c0 : cpu_clock(): 0.00 : ffff80001011e414: bl ffff80001017af20 : select_idle_cpu(): : time = cpu_clock(this) - time; 0.00 : ffff80001011e418: sub x0, x0, x25 : cost = this_sd->avg_scan_cost; 0.00 : ffff80001011e41c: ldr x2, [x26, #96] : delta = (s64)(time - cost) / 8; 0.00 : ffff80001011e420: sub x0, x0, x2 0.00 : ffff80001011e424: add x1, x0, #0x7 0.00 : ffff80001011e428: cmp x0, #0x0 0.00 : ffff80001011e42c: csel x0, x1, x0, lt // lt = tstop : this_sd->avg_scan_cost += delta; 0.00 : ffff80001011e430: add x0, x2, x0, asr #3 0.00 : ffff80001011e434: str x0, [x26, #96] : select_idle_sibling(): : i = select_idle_smt(p, target); : if ((unsigned)i < nr_cpumask_bits) : return i; : : return target; : } 0.00 : ffff80001011e438: mov w0, w19 : select_idle_cpu(): : this_sd->avg_scan_cost += delta; 0.00 : ffff80001011e43c: ldp x22, x23, [x29, #40] 0.00 : ffff80001011e440: ldp x24, x25, [x29, #56] 0.00 : ffff80001011e444: ldr x26, [x29, #72] 0.00 : ffff80001011e448: ldr x28, [x29, #88] : select_idle_sibling(): : } 0.00 : ffff80001011e44c: ldp x19, x20, [sp, #16] 0.00 : ffff80001011e450: ldr x21, [sp, #32] 0.00 : ffff80001011e454: ldr x27, [sp, #80] 0.00 : ffff80001011e458: ldp x29, x30, [sp], #96 0.00 : ffff80001011e45c: ret : cpu_clock(): 0.00 : ffff80001011e460: bl ffff80001017af20 : select_idle_cpu(): : time = cpu_clock(this) - time; 0.00 : ffff80001011e464: sub x0, x0, x25 : cost = this_sd->avg_scan_cost; 0.00 : ffff80001011e468: ldr x2, [x26, #96] : delta = (s64)(time - cost) / 8; 0.00 : ffff80001011e46c: sub x1, x0, x2 0.00 : ffff80001011e470: add x0, x1, #0x7 0.00 : ffff80001011e474: cmp x1, #0x0 0.00 : ffff80001011e478: csel x0, x0, x1, lt // lt = tstop : this_sd->avg_scan_cost += delta; 0.00 : ffff80001011e47c: add x0, x2, x0, asr #3 0.00 : ffff80001011e480: str x0, [x26, #96] 0.00 : ffff80001011e484: nop 0.00 : ffff80001011e488: ldp x22, x23, [x29, #40] 0.00 : ffff80001011e48c: ldp x24, x25, [x29, #56] 0.00 : ffff80001011e490: ldr x26, [x29, #72] 0.00 : ffff80001011e494: ldr x28, [x29, #88] 0.74 : ffff80001011e498: mov w19, w21 : select_idle_sibling(): : } 0.19 : ffff80001011e49c: mov w0, w19 14.67 : ffff80001011e4a0: ldr x21, [sp, #32] 15.93 : ffff80001011e4a4: ldp x19, x20, [sp, #16] 10.80 : ffff80001011e4a8: ldr x27, [sp, #80] 4.61 : ffff80001011e4ac: ldp x29, x30, [sp], #96 0.00 : ffff80001011e4b0: ret : if (prev != target && cpus_share_cache(prev, target) && 0.00 : ffff80001011e4b4: mov w1, w21 0.00 : ffff80001011e4b8: mov w0, w20 0.00 : ffff80001011e4bc: bl ffff800010117770 0.00 : ffff80001011e4c0: tst w0, #0xff 0.00 : ffff80001011e4c4: b.eq ffff80001011e2e4 // b.none : (available_idle_cpu(prev) || sched_idle_cpu(prev))) 0.00 : ffff80001011e4c8: mov w0, w20 0.00 : ffff80001011e4cc: bl ffff800010118870 : if (prev != target && cpus_share_cache(prev, target) && 0.00 : ffff80001011e4d0: cbnz w0, ffff80001011e4f0 : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff80001011e4d4: ldr x0, [x25, w20, sxtw #3] 0.00 : ffff80001011e4d8: add x19, x19, x0 0.00 : ffff80001011e4dc: ldr w0, [x19, #4] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff80001011e4e0: ldr w1, [x19, #160] 0.00 : ffff80001011e4e4: cmp w0, #0x0 0.00 : ffff80001011e4e8: ccmp w1, w0, #0x0, ne // ne = any 0.00 : ffff80001011e4ec: b.ne ffff80001011e2e4 // b.any : select_idle_cpu(): : this_sd->avg_scan_cost += delta; 0.00 : ffff80001011e4f0: mov w19, w20 0.00 : ffff80001011e4f4: ldp x22, x23, [x29, #40] 0.00 : ffff80001011e4f8: ldp x24, x25, [x29, #56] 0.00 : ffff80001011e4fc: b ffff80001011e49c 0.00 : ffff80001011e500: ldp x22, x23, [x29, #40] 0.00 : ffff80001011e504: ldp x24, x25, [x29, #56] 0.00 : ffff80001011e508: b ffff80001011e498 : div_u64_rem(): : * divide. : */ : static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) : { : *remainder = dividend % divisor; : return dividend / divisor; 0.00 : ffff80001011e50c: and x2, x2, #0xffffffff 0.00 : ffff80001011e510: udiv x20, x20, x2 : cpu_clock(): 0.00 : ffff80001011e514: bl ffff80001017af20 : bitmap_and(): 0.00 : ffff80001011e518: ldr x2, [x27, #736] 0.00 : ffff80001011e51c: mov x1, x19 0.00 : ffff80001011e520: mov w3, #0x100 // #256 : cpu_clock(): 0.00 : ffff80001011e524: mov x25, x0 : bitmap_and(): 0.00 : ffff80001011e528: mov x0, x22 0.00 : ffff80001011e52c: bl ffff8000104850b8 <__bitmap_and> : select_idle_cpu(): : for_each_cpu_wrap(cpu, cpus, target) { 0.00 : ffff80001011e530: mov w3, #0x0 // #0 0.00 : ffff80001011e534: mov w2, w21 0.00 : ffff80001011e538: mov x1, x22 0.00 : ffff80001011e53c: mov w0, w28 0.00 : ffff80001011e540: bl ffff800010cb9368 0.00 : ffff80001011e544: mov w19, w0 0.00 : ffff80001011e548: cmp w0, #0xff 0.00 : ffff80001011e54c: b.hi ffff80001011e5e4 // b.pmore : if (!--nr) 0.00 : ffff80001011e550: subs w20, w20, #0x1 0.00 : ffff80001011e554: b.ne ffff80001011e3b4 // b.any 0.00 : ffff80001011e558: b ffff80001011e488 : select_idle_sibling(): : prev == smp_processor_id() && 0.00 : ffff80001011e55c: adrp x0, ffff80001151d000 0.00 : ffff80001011e560: add x0, x0, #0x18 : __my_cpu_offset(): 0.00 : ffff80001011e564: mrs x1, tpidr_el1 : select_idle_sibling(): : if (is_per_cpu_kthread(current) && 0.00 : ffff80001011e568: ldr w0, [x0, x1] 0.00 : ffff80001011e56c: cmp w20, w0 0.00 : ffff80001011e570: b.ne ffff80001011e2fc // b.any : this_rq()->nr_running <= 1) { 0.00 : ffff80001011e574: add x0, x24, #0xe80 0.00 : ffff80001011e578: add x0, x0, x1 : prev == smp_processor_id() && 0.00 : ffff80001011e57c: ldr w0, [x0, #4] 0.00 : ffff80001011e580: cmp w0, #0x1 0.00 : ffff80001011e584: b.hi ffff80001011e2fc // b.pmore 0.00 : ffff80001011e588: b ffff80001011e4f0 : cpus_share_cache(recent_used_cpu, target) && 0.00 : ffff80001011e58c: mov w1, w21 0.00 : ffff80001011e590: mov w0, w19 0.00 : ffff80001011e594: bl ffff800010117770 : recent_used_cpu != target && 0.00 : ffff80001011e598: tst w0, #0xff 0.00 : ffff80001011e59c: b.eq ffff80001011e30c // b.none : (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 0.00 : ffff80001011e5a0: mov w0, w19 0.00 : ffff80001011e5a4: bl ffff800010118870 : cpus_share_cache(recent_used_cpu, target) && 0.00 : ffff80001011e5a8: cbz w0, ffff80001011e60c : cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr)) { 0.00 : ffff80001011e5ac: ldr w1, [x27, #96] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001011e5b0: ldr x2, [x27, #736] 0.00 : ffff80001011e5b4: add w0, w1, #0x3f 0.00 : ffff80001011e5b8: cmp w1, #0x0 0.00 : ffff80001011e5bc: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff80001011e5c0: asr w0, w0, #6 0.00 : ffff80001011e5c4: sxtw x0, w0 0.00 : ffff80001011e5c8: ldr x0, [x2, x0, lsl #3] 0.00 : ffff80001011e5cc: lsr x1, x0, x1 : select_idle_sibling(): : (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 0.00 : ffff80001011e5d0: tbz w1, #0, ffff80001011e30c : p->recent_used_cpu = prev; 0.00 : ffff80001011e5d4: str w20, [x27, #96] 0.00 : ffff80001011e5d8: ldp x22, x23, [x29, #40] 0.00 : ffff80001011e5dc: ldp x24, x25, [x29, #56] 0.00 : ffff80001011e5e0: b ffff80001011e49c : cpu_clock(): 0.00 : ffff80001011e5e4: bl ffff80001017af20 : select_idle_cpu(): : time = cpu_clock(this) - time; 0.00 : ffff80001011e5e8: sub x25, x0, x25 : cost = this_sd->avg_scan_cost; 0.00 : ffff80001011e5ec: ldr x1, [x26, #96] : this_sd->avg_scan_cost += delta; 0.00 : ffff80001011e5f0: mov w19, w21 : delta = (s64)(time - cost) / 8; 0.00 : ffff80001011e5f4: sub x25, x25, x1 0.00 : ffff80001011e5f8: add x0, x25, #0x7 0.00 : ffff80001011e5fc: cmp x25, #0x0 0.00 : ffff80001011e600: csel x0, x0, x25, lt // lt = tstop : this_sd->avg_scan_cost += delta; 0.00 : ffff80001011e604: add x0, x1, x0, asr #3 0.00 : ffff80001011e608: b ffff80001011e434 : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff80001011e60c: add x1, x23, #0x928 0.00 : ffff80001011e610: add x0, x24, #0xe80 0.00 : ffff80001011e614: ldr x1, [x1, w19, sxtw #3] 0.00 : ffff80001011e618: add x0, x0, x1 0.00 : ffff80001011e61c: ldr w1, [x0, #4] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff80001011e620: ldr w0, [x0, #160] 0.00 : ffff80001011e624: cmp w1, #0x0 0.00 : ffff80001011e628: ccmp w0, w1, #0x0, ne // ne = any 0.00 : ffff80001011e62c: b.ne ffff80001011e30c // b.any 0.00 : ffff80001011e630: b ffff80001011e5ac 0.00 : ffff80001011e634: ldp x22, x23, [x29, #40] 0.00 : ffff80001011e638: ldp x24, x25, [x29, #56] 0.00 : ffff80001011e63c: ldr x26, [x29, #72] 0.00 : ffff80001011e640: b ffff80001011e498 Percent | Source code & Disassembly of vmlinux for cycles (1696 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd24e0 <__schedule>: : __schedule(): : * - return from interrupt-handler to user-space : * : * WARNING: must be called with preemption disabled! : */ : static void __sched notrace __schedule(bool preempt) : { 0.11 : ffff800010cd24e0: stp x29, x30, [sp, #-96]! 0.00 : ffff800010cd24e4: mov x29, sp 0.19 : ffff800010cd24e8: stp x19, x20, [sp, #16] : struct rq_flags rf; : struct rq *rq; : int cpu; : : cpu = smp_processor_id(); : rq = cpu_rq(cpu); 0.00 : ffff800010cd24ec: adrp x19, ffff800011528000 : { 0.00 : ffff800010cd24f0: stp x21, x22, [sp, #32] 0.00 : ffff800010cd24f4: adrp x22, ffff800011909000 0.28 : ffff800010cd24f8: stp x23, x24, [sp, #48] : cpu = smp_processor_id(); 0.00 : ffff800010cd24fc: adrp x23, ffff80001151d000 : { 0.71 : ffff800010cd2500: stp x25, x26, [sp, #64] : cpu = smp_processor_id(); 0.00 : ffff800010cd2504: add x1, x23, #0x18 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.06 : ffff800010cd2508: mrs x2, tpidr_el1 : __schedule(): : rq = cpu_rq(cpu); 0.00 : ffff800010cd250c: ldrsw x2, [x1, x2] : { 0.00 : ffff800010cd2510: add x3, x22, #0x908 : rq = cpu_rq(cpu); 0.12 : ffff800010cd2514: adrp x1, ffff800011909000 0.00 : ffff800010cd2518: add x1, x1, #0x928 : { 0.30 : ffff800010cd251c: ldr x4, [x3] 0.06 : ffff800010cd2520: str x4, [x29, #88] 0.00 : ffff800010cd2524: mov x4, #0x0 // #0 : rq = cpu_rq(cpu); 0.00 : ffff800010cd2528: add x19, x19, #0xe80 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010cd252c: mrs x21, sp_el0 : __schedule(): 0.00 : ffff800010cd2530: ldr x2, [x1, x2, lsl #3] 0.06 : ffff800010cd2534: mov x25, x30 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.08 : ffff800010cd2538: ldr w1, [x21, #16] : __schedule(): : { 0.34 : ffff800010cd253c: and w20, w0, #0xff : rq = cpu_rq(cpu); 0.00 : ffff800010cd2540: add x19, x19, x2 : schedule_debug(): : if (unlikely(in_atomic_preempt_off())) { 0.00 : ffff800010cd2544: cmp w1, #0x1 : __schedule(): : prev = rq->curr; 0.00 : ffff800010cd2548: ldr x26, [x19, #2352] : schedule_debug(): : if (unlikely(in_atomic_preempt_off())) { 0.00 : ffff800010cd254c: b.ne ffff800010cd2a10 <__schedule+0x530> // b.any : profile_hit(): : static inline void profile_hit(int type, void *ip) : { : /* : * Speedup for the common (no profiling enabled) case: : */ : if (unlikely(prof_on == type)) 3.27 : ffff800010cd2550: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010cd2554: ldr w0, [x0, #696] 0.00 : ffff800010cd2558: cmp w0, #0x2 0.00 : ffff800010cd255c: b.eq ffff800010cd2a30 <__schedule+0x550> // b.none : arch_local_irq_disable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.28 : ffff800010cd2560: mov x0, #0x60 // #96 0.06 : ffff800010cd2564: msr daifset, #0x2 : __schedule(): : : if (sched_feat(HRTICK)) : hrtick_clear(rq); : : local_irq_disable(); : rcu_note_context_switch(preempt); 17.86 : ffff800010cd2568: mov w0, w20 0.00 : ffff800010cd256c: bl ffff800010160440 : rq_lock(): : : static inline void : rq_lock(struct rq *rq, struct rq_flags *rf) : __acquires(rq->lock) : { : raw_spin_lock(&rq->lock); 0.89 : ffff800010cd2570: mov x0, x19 0.00 : ffff800010cd2574: bl ffff800010cd80f0 <_raw_spin_lock> : __schedule(): : * : * The membarrier system call requires a full memory barrier : * after coming from user-space, before storing to rq->curr. : */ : rq_lock(rq, &rf); : smp_mb__after_spinlock(); 0.95 : ffff800010cd2578: dmb ish : : /* Promote REQ to ACT */ : rq->clock_update_flags <<= 1; 7.88 : ffff800010cd257c: ldr w0, [x19, #2392] 0.00 : ffff800010cd2580: lsl w0, w0, #1 0.00 : ffff800010cd2584: str w0, [x19, #2392] : update_rq_clock(): : if (rq->clock_update_flags & RQCF_ACT_SKIP) 0.00 : ffff800010cd2588: tbz w0, #1, ffff800010cd2890 <__schedule+0x3b0> : __schedule(): : update_rq_clock(rq); : : switch_count = &prev->nivcsw; 0.42 : ffff800010cd258c: add x24, x26, #0x588 : if (!preempt && prev->state) { 0.00 : ffff800010cd2590: cbnz w20, ffff800010cd259c <__schedule+0xbc> 0.00 : ffff800010cd2594: ldr x0, [x26, #24] 0.00 : ffff800010cd2598: cbnz x0, ffff800010cd2828 <__schedule+0x348> : pick_next_task(): : if (likely((prev->sched_class == &idle_sched_class || 0.46 : ffff800010cd259c: ldr x20, [x26, #128] 0.00 : ffff800010cd25a0: adrp x0, ffff800010d06000 0.00 : ffff800010cd25a4: add x0, x0, #0x128 0.00 : ffff800010cd25a8: adrp x21, ffff800010d06000 0.42 : ffff800010cd25ac: cmp x20, x0 0.00 : ffff800010cd25b0: add x0, x21, #0x60 0.00 : ffff800010cd25b4: ccmp x20, x0, #0x4, ne // ne = any 0.00 : ffff800010cd25b8: b.ne ffff800010cd2984 <__schedule+0x4a4> // b.any 0.37 : ffff800010cd25bc: ldr w1, [x19, #4] 0.89 : ffff800010cd25c0: ldr w0, [x19, #156] 0.00 : ffff800010cd25c4: cmp w1, w0 0.00 : ffff800010cd25c8: b.ne ffff800010cd2978 <__schedule+0x498> // b.any : p = pick_next_task_fair(rq, prev, rf); 0.00 : ffff800010cd25cc: add x2, x29, #0x50 0.00 : ffff800010cd25d0: mov x1, x26 0.00 : ffff800010cd25d4: mov x0, x19 0.00 : ffff800010cd25d8: bl ffff800010127a00 0.00 : ffff800010cd25dc: mov x20, x0 : if (unlikely(p == RETRY_TASK)) 0.00 : ffff800010cd25e0: cmn x0, #0x1 0.00 : ffff800010cd25e4: b.eq ffff800010cd2974 <__schedule+0x494> // b.none : if (!p) { 0.00 : ffff800010cd25e8: cbz x0, ffff800010cd28f4 <__schedule+0x414> : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010cd25ec: b ffff800010cd263c <__schedule+0x15c> 0.65 : ffff800010cd25f0: b ffff800010cd263c <__schedule+0x15c> : __lse_atomic64_andnot(): : " " #asm_op " %[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) 0.92 : ffff800010cd25f4: mov x0, #0x2 // #2 0.00 : ffff800010cd25f8: stclr x0, [x26] : get_current(): 9.98 : ffff800010cd25fc: mrs x0, sp_el0 : clear_preempt_need_resched(): : current_thread_info()->preempt.need_resched = 0; : } : : static inline void clear_preempt_need_resched(void) : { : current_thread_info()->preempt.need_resched = 1; 0.00 : ffff800010cd2600: mov w1, #0x1 // #1 0.00 : ffff800010cd2604: str w1, [x0, #20] : __schedule(): : : next = pick_next_task(rq, prev, &rf); : clear_tsk_need_resched(prev); : clear_preempt_need_resched(); : : if (likely(prev != next)) { 0.00 : ffff800010cd2608: cmp x26, x20 0.00 : ffff800010cd260c: b.ne ffff800010cd2658 <__schedule+0x178> // b.any : trace_sched_switch(preempt, prev, next); : : /* Also unlocks the rq: */ : rq = context_switch(rq, prev, next, &rf); : } else { : rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 0.00 : ffff800010cd2610: ldr w1, [x19, #2392] : rq_unlock_irq(): : static inline void : rq_unlock_irq(struct rq *rq, struct rq_flags *rf) : __releases(rq->lock) : { : rq_unpin_lock(rq, rf); : raw_spin_unlock_irq(&rq->lock); 0.00 : ffff800010cd2614: mov x0, x19 : __schedule(): 0.00 : ffff800010cd2618: and w1, w1, #0xfffffffc 0.00 : ffff800010cd261c: str w1, [x19, #2392] : rq_unlock_irq(): 0.00 : ffff800010cd2620: bl ffff800010cd7c70 <_raw_spin_unlock_irq> : balance_callback(): : if (unlikely(rq->balance_callback)) 0.00 : ffff800010cd2624: ldr x0, [x19, #2496] 0.00 : ffff800010cd2628: cbz x0, ffff800010cd2740 <__schedule+0x260> 0.00 : ffff800010cd262c: nop : __balance_callback(rq); 0.00 : ffff800010cd2630: mov x0, x19 0.00 : ffff800010cd2634: bl ffff800010113728 <__balance_callback> : __schedule(): : rq_unlock_irq(rq, &rf); : } : : balance_callback(rq); : } 0.00 : ffff800010cd2638: b ffff800010cd2740 <__schedule+0x260> : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff800010cd263c: mov x0, #0x2 // #2 0.00 : ffff800010cd2640: b ffff800010cd303c : get_current(): 0.00 : ffff800010cd2644: mrs x0, sp_el0 : clear_preempt_need_resched(): 0.00 : ffff800010cd2648: mov w1, #0x1 // #1 0.00 : ffff800010cd264c: str w1, [x0, #20] : __schedule(): : if (likely(prev != next)) { 0.00 : ffff800010cd2650: cmp x26, x20 0.00 : ffff800010cd2654: b.eq ffff800010cd2610 <__schedule+0x130> // b.none : rq->nr_switches++; 0.23 : ffff800010cd2658: ldr x0, [x19, #64] : sched_info_on(): : { : #ifdef CONFIG_SCHEDSTATS : return 1; : #elif defined(CONFIG_TASK_DELAY_ACCT) : extern int delayacct_on; : return delayacct_on; 0.00 : ffff800010cd265c: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 1.54 : ffff800010cd2660: str x20, [x19, #2352] : __schedule(): 0.00 : ffff800010cd2664: add x0, x0, #0x1 0.11 : ffff800010cd2668: str x0, [x19, #64] : ++*switch_count; 0.00 : ffff800010cd266c: ldr x0, [x24] 0.00 : ffff800010cd2670: add x0, x0, #0x1 0.59 : ffff800010cd2674: str x0, [x24] : sched_info_switch(): : } : : static inline void : sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) : { : if (sched_info_on()) 0.35 : ffff800010cd2678: ldr w0, [x1, #792] 0.00 : ffff800010cd267c: cbnz w0, ffff800010cd27bc <__schedule+0x2dc> : arch_static_branch(): : asm_volatile_goto( 0.18 : ffff800010cd2680: nop 0.06 : ffff800010cd2684: nop : rseq_set_notify_resume(): : RSEQ_EVENT_MIGRATE = (1U << RSEQ_EVENT_MIGRATE_BIT), : }; : : static inline void rseq_set_notify_resume(struct task_struct *t) : { : if (t->rseq) 0.00 : ffff800010cd2688: ldr x1, [x26, #2304] : __set_bit(): : static inline void __set_bit(int nr, volatile unsigned long *addr) : { : unsigned long mask = BIT_MASK(nr); : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); : : *p |= mask; 3.12 : ffff800010cd268c: ldr x0, [x26, #2320] 0.00 : ffff800010cd2690: orr x0, x0, #0x1 1.13 : ffff800010cd2694: str x0, [x26, #2320] : rseq_set_notify_resume(): 0.00 : ffff800010cd2698: cbz x1, ffff800010cd26ac <__schedule+0x1cc> : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff800010cd269c: b ffff800010cd2860 <__schedule+0x380> 0.00 : ffff800010cd26a0: b ffff800010cd2860 <__schedule+0x380> : __lse_atomic64_or(): : ATOMIC64_OP(or, stset) 0.00 : ffff800010cd26a4: mov x0, #0x4 // #4 0.00 : ffff800010cd26a8: stset x0, [x26] : arch_static_branch(): : asm_volatile_goto( 0.53 : ffff800010cd26ac: nop : context_switch(): : if (!next->mm) { // to kernel 0.24 : ffff800010cd26b0: ldr x0, [x20, #952] : prepare_task(): : next->on_cpu = 1; 0.00 : ffff800010cd26b4: mov w1, #0x1 // #1 3.64 : ffff800010cd26b8: str w1, [x20, #64] : context_switch(): : if (!next->mm) { // to kernel 0.00 : ffff800010cd26bc: cbz x0, ffff800010cd289c <__schedule+0x3bc> : membarrier_switch_mm(): : struct mm_struct *prev_mm, : struct mm_struct *next_mm) : { : int membarrier_state; : : if (prev_mm == next_mm) 0.00 : ffff800010cd26c0: ldr x1, [x26, #960] 0.00 : ffff800010cd26c4: cmp x0, x1 0.00 : ffff800010cd26c8: b.eq ffff800010cd2710 <__schedule+0x230> // b.none : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd26cc: ldr w1, [x0, #72] 0.00 : ffff800010cd26d0: ldr w2, [x19, #2460] : membarrier_switch_mm(): : return; : : membarrier_state = atomic_read(&next_mm->membarrier_state); : if (READ_ONCE(rq->membarrier_state) == membarrier_state) 0.00 : ffff800010cd26d4: cmp w1, w2 0.00 : ffff800010cd26d8: b.eq ffff800010cd26f0 <__schedule+0x210> // b.none : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010cd26dc: str w1, [x19, #2460] : context_switch(): : switch_mm_irqs_off(prev->active_mm, next->mm, next); 0.00 : ffff800010cd26e0: ldr x0, [x20, #952] : switch_mm(): : : static inline void : switch_mm(struct mm_struct *prev, struct mm_struct *next, : struct task_struct *tsk) : { : if (prev != next) 0.00 : ffff800010cd26e4: ldr x1, [x26, #960] 0.00 : ffff800010cd26e8: cmp x1, x0 0.00 : ffff800010cd26ec: b.eq ffff800010cd2710 <__schedule+0x230> // b.none : __switch_mm(): : if (next == &init_mm) { 0.00 : ffff800010cd26f0: adrp x1, ffff80001192c000 0.00 : ffff800010cd26f4: add x1, x1, #0x3a0 : unsigned int cpu = smp_processor_id(); 0.00 : ffff800010cd26f8: add x23, x23, #0x18 : if (next == &init_mm) { 0.00 : ffff800010cd26fc: cmp x0, x1 : __my_cpu_offset(): 0.00 : ffff800010cd2700: mrs x2, tpidr_el1 : __switch_mm(): 0.00 : ffff800010cd2704: b.eq ffff800010cd28d4 <__schedule+0x3f4> // b.none : check_and_switch_context(next, cpu); 0.00 : ffff800010cd2708: ldr w1, [x23, x2] 0.00 : ffff800010cd270c: bl ffff8000100a45f8 : context_switch(): : if (!prev->mm) { // from kernel 0.26 : ffff800010cd2710: ldr x0, [x26, #952] 0.00 : ffff800010cd2714: cbz x0, ffff800010cd28c4 <__schedule+0x3e4> : rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 2.27 : ffff800010cd2718: ldr w2, [x19, #2392] : switch_to(prev, next, prev); 0.00 : ffff800010cd271c: mov x1, x20 0.06 : ffff800010cd2720: mov x0, x26 : rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 0.00 : ffff800010cd2724: and w2, w2, #0xfffffffc 1.57 : ffff800010cd2728: str w2, [x19, #2392] : switch_to(prev, next, prev); 0.00 : ffff800010cd272c: bl ffff800010087c40 <__switch_to> : return finish_task_switch(prev); 0.83 : ffff800010cd2730: bl ffff800010114248 0.00 : ffff800010cd2734: mov x19, x0 : balance_callback(): : if (unlikely(rq->balance_callback)) 0.00 : ffff800010cd2738: ldr x0, [x19, #2496] 0.00 : ffff800010cd273c: cbnz x0, ffff800010cd2630 <__schedule+0x150> : __schedule(): : } 0.06 : ffff800010cd2740: add x22, x22, #0x908 0.30 : ffff800010cd2744: ldr x1, [x29, #88] 0.11 : ffff800010cd2748: ldr x0, [x22] 0.00 : ffff800010cd274c: eor x0, x1, x0 0.00 : ffff800010cd2750: cbnz x0, ffff800010cd2a70 <__schedule+0x590> 0.25 : ffff800010cd2754: ldp x19, x20, [sp, #16] 0.46 : ffff800010cd2758: ldp x21, x22, [sp, #32] 0.12 : ffff800010cd275c: ldp x23, x24, [sp, #48] 0.00 : ffff800010cd2760: ldp x25, x26, [sp, #64] 0.00 : ffff800010cd2764: ldp x29, x30, [sp], #96 0.00 : ffff800010cd2768: ret : perf_event_task_sched_out(): : struct task_struct *next) : { : perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0); : : if (static_branch_unlikely(&perf_sched_events)) : __perf_event_task_sched_out(prev, next); 0.00 : ffff800010cd276c: mov x1, x20 0.00 : ffff800010cd2770: mov x0, x26 0.00 : ffff800010cd2774: bl ffff8000101c4a88 <__perf_event_task_sched_out> 0.00 : ffff800010cd2778: b ffff800010cd2688 <__schedule+0x1a8> : __my_cpu_offset(): : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff800010cd277c: mov x5, sp : perf_sw_event_sched(): : struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]); 0.00 : ffff800010cd2780: adrp x4, ffff800011526000 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010cd2784: mrs x0, tpidr_el1 : perf_sw_event_sched(): 0.00 : ffff800010cd2788: add x4, x4, #0x150 : perf_fetch_caller_regs(): : perf_arch_fetch_caller_regs(regs, CALLER_ADDR0); 0.00 : ffff800010cd278c: mov x6, #0x5 // #5 : perf_sw_event_sched(): : struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]); 0.00 : ffff800010cd2790: add x4, x4, x0 : ___perf_sw_event(event_id, nr, regs, addr); 0.00 : ffff800010cd2794: mov x3, #0x0 // #0 0.00 : ffff800010cd2798: mov x2, x4 0.00 : ffff800010cd279c: mov x1, #0x1 // #1 0.00 : ffff800010cd27a0: mov w0, #0x3 // #3 : perf_fetch_caller_regs(): : perf_arch_fetch_caller_regs(regs, CALLER_ADDR0); 0.00 : ffff800010cd27a4: str x29, [x4, #232] 0.00 : ffff800010cd27a8: stp x5, x25, [x4, #248] 0.00 : ffff800010cd27ac: str x6, [x4, #264] : perf_sw_event_sched(): : ___perf_sw_event(event_id, nr, regs, addr); 0.00 : ffff800010cd27b0: bl ffff8000101ca150 <___perf_sw_event> : arch_static_branch(): 0.00 : ffff800010cd27b4: nop 0.00 : ffff800010cd27b8: b ffff800010cd2688 <__schedule+0x1a8> : __sched_info_switch(): : if (prev != rq->idle) 0.88 : ffff800010cd27bc: ldr x0, [x19, #2360] 0.36 : ffff800010cd27c0: cmp x26, x0 0.00 : ffff800010cd27c4: b.eq ffff800010cd27f0 <__schedule+0x310> // b.none : sched_info_depart(): : if (t->state == TASK_RUNNING) 3.17 : ffff800010cd27c8: ldr x1, [x26, #24] 0.00 : ffff800010cd27cc: cbnz x1, ffff800010cd27e8 <__schedule+0x308> : sched_info_queued(): : if (!t->sched_info.last_queued) 0.57 : ffff800010cd27d0: ldr x1, [x26, #864] 0.00 : ffff800010cd27d4: cbnz x1, ffff800010cd27e8 <__schedule+0x308> : t->sched_info.last_queued = rq_clock(rq); 0.00 : ffff800010cd27d8: ldr x0, [x19, #2400] 0.00 : ffff800010cd27dc: str x0, [x26, #864] 0.00 : ffff800010cd27e0: ldr x0, [x19, #2360] 0.00 : ffff800010cd27e4: nop : __sched_info_switch(): : if (next != rq->idle) 1.80 : ffff800010cd27e8: cmp x0, x20 0.00 : ffff800010cd27ec: b.eq ffff800010cd2680 <__schedule+0x1a0> // b.none : sched_info_arrive(): : if (t->sched_info.last_queued) 0.18 : ffff800010cd27f0: ldr x0, [x20, #864] : unsigned long long now = rq_clock(rq), delta = 0; 0.81 : ffff800010cd27f4: ldr x3, [x19, #2400] : t->sched_info.run_delay += delta; 1.93 : ffff800010cd27f8: ldr x1, [x20, #848] : delta = now - t->sched_info.last_queued; 0.00 : ffff800010cd27fc: cmp x0, #0x0 0.00 : ffff800010cd2800: sub x4, x3, x0 : t->sched_info.pcount++; 0.00 : ffff800010cd2804: ldr x2, [x20, #840] : delta = now - t->sched_info.last_queued; 0.00 : ffff800010cd2808: csel x0, x4, x0, ne // ne = any : t->sched_info.run_delay += delta; 0.00 : ffff800010cd280c: add x0, x1, x0 : sched_info_reset_dequeued(): : t->sched_info.last_queued = 0; 0.00 : ffff800010cd2810: add x1, x20, #0x200 : sched_info_arrive(): : t->sched_info.pcount++; 0.00 : ffff800010cd2814: add x2, x2, #0x1 : sched_info_reset_dequeued(): : t->sched_info.last_queued = 0; 0.00 : ffff800010cd2818: stp x2, x0, [x1, #328] 0.23 : ffff800010cd281c: stp x3, xzr, [x1, #344] : arch_static_branch(): 0.00 : ffff800010cd2820: nop 0.13 : ffff800010cd2824: b ffff800010cd2684 <__schedule+0x1a4> : __schedule(): : if (signal_pending_state(prev->state, prev)) { 0.00 : ffff800010cd2828: ldr x0, [x26, #24] : signal_pending_state(): : return signal_pending(p) && __fatal_signal_pending(p); : } : : static inline int signal_pending_state(long state, struct task_struct *p) : { : if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL))) 0.00 : ffff800010cd282c: mov x1, #0x101 // #257 0.00 : ffff800010cd2830: tst x0, x1 0.00 : ffff800010cd2834: b.eq ffff800010cd2840 <__schedule+0x360> // b.none : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.24 : ffff800010cd2838: ldr x1, [x26] : signal_pending_state(): : return 0; : if (!signal_pending(p)) 0.00 : ffff800010cd283c: tbnz w1, #0, ffff800010cd2a40 <__schedule+0x560> : __schedule(): : deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); 0.42 : ffff800010cd2840: mov x0, x19 0.00 : ffff800010cd2844: mov w2, #0x9 // #9 0.00 : ffff800010cd2848: mov x1, x26 0.00 : ffff800010cd284c: bl ffff8000101150a8 : if (prev->in_iowait) { 0.00 : ffff800010cd2850: ldrb w0, [x26, #1064] 0.00 : ffff800010cd2854: tbnz w0, #1, ffff800010cd2928 <__schedule+0x448> : switch_count = &prev->nvcsw; 0.00 : ffff800010cd2858: add x24, x26, #0x580 0.00 : ffff800010cd285c: b ffff800010cd259c <__schedule+0xbc> : __ll_sc_atomic64_or(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010cd2860: b ffff800010cd3054 0.00 : ffff800010cd2864: b ffff800010cd26ac <__schedule+0x1cc> : __fire_sched_out_preempt_notifiers(): : hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 0.00 : ffff800010cd2868: ldr x21, [x26, #720] 0.00 : ffff800010cd286c: cbz x21, ffff800010cd26b0 <__schedule+0x1d0> : notifier->ops->sched_out(notifier, next); 0.00 : ffff800010cd2870: ldr x2, [x21, #16] 0.00 : ffff800010cd2874: mov x0, x21 0.00 : ffff800010cd2878: mov x1, x20 0.00 : ffff800010cd287c: ldr x2, [x2, #8] 0.00 : ffff800010cd2880: blr x2 : hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 0.00 : ffff800010cd2884: ldr x21, [x21] 0.00 : ffff800010cd2888: cbnz x21, ffff800010cd2870 <__schedule+0x390> 0.00 : ffff800010cd288c: b ffff800010cd26b0 <__schedule+0x1d0> : update_rq_clock(): 0.00 : ffff800010cd2890: mov x0, x19 0.00 : ffff800010cd2894: bl ffff800010113d48 0.12 : ffff800010cd2898: b ffff800010cd258c <__schedule+0xac> : context_switch(): : enter_lazy_tlb(prev->active_mm, next); 1.35 : ffff800010cd289c: ldr x0, [x26, #960] : next->active_mm = prev->active_mm; 1.42 : ffff800010cd28a0: str x0, [x20, #960] : if (prev->mm) // from user 0.55 : ffff800010cd28a4: ldr x0, [x26, #952] 0.00 : ffff800010cd28a8: cbz x0, ffff800010cd28cc <__schedule+0x3ec> : mmgrab(): : * See also for an in-depth explanation : * of &mm_struct.mm_count vs &mm_struct.mm_users. : */ : static inline void mmgrab(struct mm_struct *mm) : { : atomic_inc(&mm->mm_count); 0.00 : ffff800010cd28ac: ldr x0, [x26, #960] 0.00 : ffff800010cd28b0: add x2, x0, #0x50 : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff800010cd28b4: b ffff800010cd2a04 <__schedule+0x524> 0.00 : ffff800010cd28b8: b ffff800010cd2a04 <__schedule+0x524> : __lse_atomic_add(): : ATOMIC_OP(add, stadd) 0.00 : ffff800010cd28bc: stadd w1, [x2] 0.57 : ffff800010cd28c0: b ffff800010cd2718 <__schedule+0x238> : context_switch(): : rq->prev_mm = prev->active_mm; 0.19 : ffff800010cd28c4: ldr x0, [x26, #960] 0.19 : ffff800010cd28c8: str x0, [x19, #2384] : prev->active_mm = NULL; 0.36 : ffff800010cd28cc: str xzr, [x26, #960] 0.00 : ffff800010cd28d0: b ffff800010cd2718 <__schedule+0x238> : cpu_set_reserved_ttbr0(): : unsigned long ttbr = phys_to_ttbr(__pa_symbol(empty_zero_page)); 0.00 : ffff800010cd28d4: adrp x1, ffff8000112f2000 0.00 : ffff800010cd28d8: adrp x0, ffff800011af1000 0.00 : ffff800010cd28dc: add x0, x0, #0x0 0.00 : ffff800010cd28e0: ldr x1, [x1, #1904] 0.00 : ffff800010cd28e4: sub x0, x0, x1 : write_sysreg(ttbr, ttbr0_el1); 0.00 : ffff800010cd28e8: msr ttbr0_el1, x0 : isb(); 0.00 : ffff800010cd28ec: isb 0.00 : ffff800010cd28f0: b ffff800010cd2710 <__schedule+0x230> : put_prev_task(): : WARN_ON_ONCE(rq->curr != prev); 0.00 : ffff800010cd28f4: ldr x0, [x19, #2352] 0.00 : ffff800010cd28f8: cmp x26, x0 0.00 : ffff800010cd28fc: b.ne ffff800010cd2a64 <__schedule+0x584> // b.any : prev->sched_class->put_prev_task(rq, prev); 0.00 : ffff800010cd2900: ldr x2, [x26, #128] 0.00 : ffff800010cd2904: mov x1, x26 0.00 : ffff800010cd2908: mov x0, x19 0.00 : ffff800010cd290c: ldr x2, [x2, #56] 0.00 : ffff800010cd2910: blr x2 : pick_next_task(): : p = pick_next_task_idle(rq); 0.00 : ffff800010cd2914: mov x0, x19 0.00 : ffff800010cd2918: bl ffff80001011bde0 0.00 : ffff800010cd291c: mov x20, x0 : arch_static_branch_jump(): 0.00 : ffff800010cd2920: b ffff800010cd263c <__schedule+0x15c> 0.00 : ffff800010cd2924: b ffff800010cd25f0 <__schedule+0x110> : __schedule(): : atomic_inc(&rq->nr_iowait); 0.00 : ffff800010cd2928: add x0, x19, #0x998 : arch_static_branch_jump(): 0.00 : ffff800010cd292c: b ffff800010cd2940 <__schedule+0x460> 0.00 : ffff800010cd2930: b ffff800010cd2940 <__schedule+0x460> : __lse_atomic_add(): 0.00 : ffff800010cd2934: mov w1, #0x1 // #1 0.00 : ffff800010cd2938: stadd w1, [x0] 0.00 : ffff800010cd293c: b ffff800010cd2944 <__schedule+0x464> : __ll_sc_atomic_add(): : ATOMIC_OPS(add, add, I) 0.00 : ffff800010cd2940: b ffff800010cd306c : get_current(): 0.00 : ffff800010cd2944: mrs x0, sp_el0 : delayacct_set_flag(): : return 0; : } : : static inline void delayacct_set_flag(int flag) : { : if (current->delays) 0.00 : ffff800010cd2948: ldr x0, [x0, #2368] 0.00 : ffff800010cd294c: cbz x0, ffff800010cd295c <__schedule+0x47c> : current->delays->flags |= flag; 0.00 : ffff800010cd2950: ldr w1, [x0, #4] 0.00 : ffff800010cd2954: orr w1, w1, #0x2 0.00 : ffff800010cd2958: str w1, [x0, #4] : get_current(): 0.00 : ffff800010cd295c: mrs x0, sp_el0 : delayacct_blkio_start(): : } : : static inline void delayacct_blkio_start(void) : { : delayacct_set_flag(DELAYACCT_PF_BLKIO); : if (current->delays) 0.00 : ffff800010cd2960: ldr x0, [x0, #2368] 0.00 : ffff800010cd2964: cbz x0, ffff800010cd2858 <__schedule+0x378> : __delayacct_blkio_start(); 0.00 : ffff800010cd2968: bl ffff8000101b2c30 <__delayacct_blkio_start> : __schedule(): : switch_count = &prev->nvcsw; 0.00 : ffff800010cd296c: add x24, x26, #0x580 0.00 : ffff800010cd2970: b ffff800010cd259c <__schedule+0xbc> 0.00 : ffff800010cd2974: ldr x20, [x26, #128] : pick_next_task(): : for_class_range(class, prev->sched_class, &idle_sched_class) { 0.06 : ffff800010cd2978: add x0, x21, #0x60 0.00 : ffff800010cd297c: cmp x20, x0 0.00 : ffff800010cd2980: b.eq ffff800010cd29b0 <__schedule+0x4d0> // b.none 0.00 : ffff800010cd2984: add x21, x21, #0x60 0.00 : ffff800010cd2988: b ffff800010cd2998 <__schedule+0x4b8> 0.24 : ffff800010cd298c: ldr x20, [x20] 0.00 : ffff800010cd2990: cmp x20, x21 0.00 : ffff800010cd2994: b.eq ffff800010cd29b0 <__schedule+0x4d0> // b.none : if (class->balance(rq, prev, rf)) 0.13 : ffff800010cd2998: ldr x3, [x20, #72] 0.00 : ffff800010cd299c: add x2, x29, #0x50 0.00 : ffff800010cd29a0: mov x1, x26 0.00 : ffff800010cd29a4: mov x0, x19 0.42 : ffff800010cd29a8: blr x3 0.17 : ffff800010cd29ac: cbz w0, ffff800010cd298c <__schedule+0x4ac> : put_prev_task(): : WARN_ON_ONCE(rq->curr != prev); 1.01 : ffff800010cd29b0: ldr x0, [x19, #2352] 0.00 : ffff800010cd29b4: cmp x26, x0 0.06 : ffff800010cd29b8: b.ne ffff800010cd2a5c <__schedule+0x57c> // b.any : prev->sched_class->put_prev_task(rq, prev); 0.89 : ffff800010cd29bc: ldr x2, [x26, #128] : pick_next_task(): : for_each_class(class) { 0.06 : ffff800010cd29c0: adrp x21, ffff800010d06000 : put_prev_task(): 0.00 : ffff800010cd29c4: mov x1, x26 : pick_next_task(): 0.00 : ffff800010cd29c8: add x21, x21, #0x388 : put_prev_task(): 0.48 : ffff800010cd29cc: mov x0, x19 0.07 : ffff800010cd29d0: ldr x2, [x2, #56] 0.00 : ffff800010cd29d4: blr x2 : pick_next_task(): 0.00 : ffff800010cd29d8: cbnz x21, ffff800010cd29e8 <__schedule+0x508> 0.00 : ffff800010cd29dc: b ffff800010cd2a6c <__schedule+0x58c> 1.26 : ffff800010cd29e0: ldr x21, [x21] 0.00 : ffff800010cd29e4: cbz x21, ffff800010cd2a6c <__schedule+0x58c> : p = class->pick_next_task(rq); 6.71 : ffff800010cd29e8: ldr x1, [x21, #48] 0.00 : ffff800010cd29ec: mov x0, x19 0.00 : ffff800010cd29f0: blr x1 4.66 : ffff800010cd29f4: mov x20, x0 : if (p) 0.00 : ffff800010cd29f8: cbz x0, ffff800010cd29e0 <__schedule+0x500> : arch_static_branch_jump(): 0.59 : ffff800010cd29fc: b ffff800010cd263c <__schedule+0x15c> 1.12 : ffff800010cd2a00: b ffff800010cd25f0 <__schedule+0x110> : __ll_sc_atomic_add(): 0.00 : ffff800010cd2a04: add x0, x0, #0x50 0.00 : ffff800010cd2a08: b ffff800010cd3084 0.00 : ffff800010cd2a0c: b ffff800010cd2718 <__schedule+0x238> : schedule_debug(): : __schedule_bug(prev); 0.00 : ffff800010cd2a10: mov x0, x26 0.00 : ffff800010cd2a14: bl ffff800010113a38 <__schedule_bug> : __write_once_size(): 0.00 : ffff800010cd2a18: mov w0, #0x1 // #1 0.00 : ffff800010cd2a1c: str w0, [x21, #16] : profile_hit(): 0.00 : ffff800010cd2a20: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010cd2a24: ldr w0, [x0, #696] 0.00 : ffff800010cd2a28: cmp w0, #0x2 0.00 : ffff800010cd2a2c: b.ne ffff800010cd2560 <__schedule+0x80> // b.any : profile_hits(type, ip, 1); 0.00 : ffff800010cd2a30: mov w2, #0x1 // #1 0.00 : ffff800010cd2a34: mov x1, x25 0.00 : ffff800010cd2a38: bl ffff800010166e70 0.00 : ffff800010cd2a3c: b ffff800010cd2560 <__schedule+0x80> : signal_pending_state(): : return 0; : : return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); 0.00 : ffff800010cd2a40: tbnz w0, #0, ffff800010cd2a50 <__schedule+0x570> : sigismember(): : : static inline int sigismember(sigset_t *set, int _sig) : { : unsigned long sig = _sig - 1; : if (_NSIG_WORDS == 1) : return 1 & (set->sig[0] >> sig); 0.00 : ffff800010cd2a44: ldr x0, [x26, #1696] : signal_pending_state(): 0.00 : ffff800010cd2a48: tst w0, #0x100 0.00 : ffff800010cd2a4c: b.eq ffff800010cd2840 <__schedule+0x360> // b.none : __schedule(): : prev->state = TASK_RUNNING; 0.00 : ffff800010cd2a50: str xzr, [x26, #24] : switch_count = &prev->nvcsw; 0.00 : ffff800010cd2a54: add x24, x26, #0x580 0.00 : ffff800010cd2a58: b ffff800010cd259c <__schedule+0xbc> : put_prev_task(): : WARN_ON_ONCE(rq->curr != prev); 0.00 : ffff800010cd2a5c: brk #0x800 0.00 : ffff800010cd2a60: b ffff800010cd29bc <__schedule+0x4dc> 0.00 : ffff800010cd2a64: brk #0x800 0.00 : ffff800010cd2a68: b ffff800010cd2900 <__schedule+0x420> : pick_next_task(): : BUG(); 0.00 : ffff800010cd2a6c: brk #0x800 : __schedule(): : } 0.00 : ffff800010cd2a70: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (1028 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010160fc0 : : rcu_irq_exit(): : * : * If you add or remove a call to rcu_irq_exit(), be sure to test with : * CONFIG_RCU_EQS_DEBUG=y. : */ : void rcu_irq_exit(void) : { 6.62 : ffff800010160fc0: stp x29, x30, [sp, #-16]! : rcu_nmi_exit_common(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff800010160fc4: adrp x0, ffff800011529000 0.00 : ffff800010160fc8: add x1, x0, #0xac0 : rcu_irq_exit(): : { 0.00 : ffff800010160fcc: mov x29, sp : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.37 : ffff800010160fd0: mov x2, sp : asm(ALTERNATIVE("mrs %0, tpidr_el1", 24.51 : ffff800010160fd4: mrs x3, tpidr_el1 : rcu_nmi_exit_common(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.94 : ffff800010160fd8: add x1, x1, x3 : WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0); 5.66 : ffff800010160fdc: ldr x3, [x1, #208] 0.00 : ffff800010160fe0: cmp x3, #0x0 0.00 : ffff800010160fe4: b.le ffff800010161038 : rcu_dynticks_curr_cpu_in_eqs(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 1.05 : ffff800010160fe8: add x0, x0, #0xac0 : __my_cpu_offset(): 26.26 : ffff800010160fec: mrs x2, tpidr_el1 : rcu_dynticks_curr_cpu_in_eqs(): 0.67 : ffff800010160ff0: add x0, x0, x2 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010160ff4: ldr w0, [x0, #216] : rcu_nmi_exit_common(): : WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); 0.00 : ffff800010160ff8: tbz w0, #1, ffff800010161030 : if (rdp->dynticks_nmi_nesting != 1) { 1.72 : ffff800010160ffc: ldr x0, [x1, #208] 0.10 : ffff800010161000: cmp x0, #0x1 0.00 : ffff800010161004: b.eq ffff80001016101c // b.none : WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */ 29.03 : ffff800010161008: sub x0, x0, #0x2 : __read_once_size(): 1.78 : ffff80001016100c: ldr w2, [x1, #216] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.26 : ffff800010161010: str x0, [x1, #208] : rcu_irq_exit(): : lockdep_assert_irqs_disabled(); : rcu_nmi_exit_common(true); : } 0.10 : ffff800010161014: ldp x29, x30, [sp], #16 0.00 : ffff800010161018: ret : __read_once_size(): : __READ_ONCE_SIZE; 0.46 : ffff80001016101c: ldr w0, [x1, #216] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.36 : ffff800010161020: str xzr, [x1, #208] : rcu_nmi_exit_common(): : rcu_dynticks_eqs_enter(); 0.00 : ffff800010161024: bl ffff80001015ded0 : rcu_irq_exit(): : } 0.10 : ffff800010161028: ldp x29, x30, [sp], #16 0.00 : ffff80001016102c: ret : rcu_nmi_exit_common(): : WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs()); 0.00 : ffff800010161030: brk #0x800 0.00 : ffff800010161034: b ffff800010160ffc : WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0); 0.00 : ffff800010161038: brk #0x800 0.00 : ffff80001016103c: b ffff800010160fe8 Percent | Source code & Disassembly of vmlinux for cycles (1082 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010aeb5f0 : : arch_timer_reg_read_cp15(): : u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) : { : if (access == ARCH_TIMER_PHYS_ACCESS) { : switch (reg) { : case ARCH_TIMER_REG_CTRL: : return read_sysreg(cntp_ctl_el0); 0.00 : ffff800010aeb5f0: mrs x1, cntp_ctl_el0 : set_next_event(): : static __always_inline void set_next_event(const int access, unsigned long evt, : struct clock_event_device *clk) : { : unsigned long ctrl; : ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); : ctrl |= ARCH_TIMER_CTRL_ENABLE; 0.75 : ffff800010aeb5f4: orr w1, w1, #0x1 : arch_timer_reg_write_cp15(): : write_sysreg(val, cntp_tval_el0); 0.00 : ffff800010aeb5f8: and x0, x0, #0xffffffff 1.28 : ffff800010aeb5fc: msr cntp_tval_el0, x0 : isb(); 0.00 : ffff800010aeb600: isb : set_next_event(): : ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; 49.80 : ffff800010aeb604: and x0, x1, #0xfffffffffffffffd : arch_timer_reg_write_cp15(): : write_sysreg(val, cntp_ctl_el0); 1.48 : ffff800010aeb608: msr cntp_ctl_el0, x0 : isb(); 0.00 : ffff800010aeb60c: isb : arch_timer_set_next_event_phys(): : static int arch_timer_set_next_event_phys(unsigned long evt, : struct clock_event_device *clk) : { : set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); : return 0; : } 46.69 : ffff800010aeb610: mov w0, #0x0 // #0 0.00 : ffff800010aeb614: ret Percent | Source code & Disassembly of vmlinux for cycles (1049 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010161098 : : rcu_idle_exit(): : * : * If you add or remove a call to rcu_idle_exit(), be sure to test with : * CONFIG_RCU_EQS_DEBUG=y. : */ : void rcu_idle_exit(void) : { 0.09 : ffff800010161098: stp x29, x30, [sp, #-16]! 0.00 : ffff80001016109c: mov x29, sp : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 1.43 : ffff8000101610a0: mrs x6, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000101610a4: and w0, w6, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff8000101610a8: cbnz w0, ffff8000101610b4 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff8000101610ac: mov x0, #0x60 // #96 0.00 : ffff8000101610b0: msr daifset, #0x2 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 9.38 : ffff8000101610b4: mrs x0, tpidr_el1 : rcu_eqs_exit(): : rdp = this_cpu_ptr(&rcu_data); 1.35 : ffff8000101610b8: adrp x5, ffff800011529000 0.00 : ffff8000101610bc: add x5, x5, #0xac0 0.00 : ffff8000101610c0: add x5, x5, x0 : oldval = rdp->dynticks_nesting; 0.00 : ffff8000101610c4: ldr x0, [x5, #200] : if (oldval) { 0.00 : ffff8000101610c8: cbz x0, ffff8000101610f0 : rdp->dynticks_nesting++; 0.00 : ffff8000101610cc: add x0, x0, #0x1 0.00 : ffff8000101610d0: str x0, [x5, #200] : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff8000101610d4: msr daif, x6 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 15.08 : ffff8000101610d8: nop : rcu_idle_exit(): : unsigned long flags; : : local_irq_save(flags); : rcu_eqs_exit(false); : local_irq_restore(flags); : } 0.00 : ffff8000101610dc: ldp x29, x30, [sp], #16 0.00 : ffff8000101610e0: ret : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.48 : ffff8000101610e4: dsb sy : rcu_idle_exit(): 50.12 : ffff8000101610e8: ldp x29, x30, [sp], #16 0.00 : ffff8000101610ec: ret : rcu_eqs_exit(): : rcu_dynticks_eqs_exit(); 8.62 : ffff8000101610f0: bl ffff80001015df68 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 7.12 : ffff8000101610f4: ldr w1, [x5, #216] : rcu_eqs_exit(): : WARN_ON_ONCE(rdp->dynticks_nmi_nesting); 6.13 : ffff8000101610f8: ldr x0, [x5, #208] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101610fc: mov x1, #0x1 // #1 0.00 : ffff800010161100: str x1, [x5, #200] : rcu_eqs_exit(): 0.00 : ffff800010161104: cbnz x0, ffff800010161114 : __write_once_size(): 0.19 : ffff800010161108: mov x0, #0x4000000000000000 // #4611686018427387904 0.00 : ffff80001016110c: str x0, [x5, #208] 0.00 : ffff800010161110: b ffff8000101610d4 : rcu_eqs_exit(): 0.00 : ffff800010161114: brk #0x800 0.00 : ffff800010161118: b ffff800010161108 Percent | Source code & Disassembly of vmlinux for cycles (971 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011bf70 : : do_idle(): : * Generic idle loop implementation : * : * Called with polling cleared. : */ : static void do_idle(void) : { 2.88 : ffff80001011bf70: stp x29, x30, [sp, #-112]! : int cpu = smp_processor_id(); 0.00 : ffff80001011bf74: adrp x0, ffff80001151d000 0.00 : ffff80001011bf78: add x0, x0, #0x18 : { 0.00 : ffff80001011bf7c: mov x29, sp 9.01 : ffff80001011bf80: str x20, [sp, #24] 1.24 : ffff80001011bf84: str x22, [sp, #40] 0.00 : ffff80001011bf88: adrp x22, ffff800011909000 0.00 : ffff80001011bf8c: add x1, x22, #0x908 0.00 : ffff80001011bf90: ldr x2, [x1] 1.26 : ffff80001011bf94: str x2, [x29, #104] 0.00 : ffff80001011bf98: mov x2, #0x0 // #0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011bf9c: mrs x1, tpidr_el1 : do_idle(): : int cpu = smp_processor_id(); 0.00 : ffff80001011bfa0: ldr w20, [x0, x1] : * then setting need_resched is guaranteed to cause the CPU to : * reschedule. : */ : : __current_set_polling(); : tick_nohz_idle_enter(); 0.00 : ffff80001011bfa4: bl ffff80001017bdf8 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.10 : ffff80001011bfa8: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.31 : ffff80001011bfac: ldr x0, [x0] : do_idle(): : : while (!need_resched()) { 0.00 : ffff80001011bfb0: tst w0, #0x2 0.00 : ffff80001011bfb4: b.ne ffff80001011c0f0 // b.any : test_bit(): 0.00 : ffff80001011bfb8: cmp w20, #0x0 0.00 : ffff80001011bfbc: str x19, [x29, #16] 0.00 : ffff80001011bfc0: add w19, w20, #0x3f 0.00 : ffff80001011bfc4: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001011bfc8: csel w19, w19, w20, lt // lt = tstop 0.61 : ffff80001011bfcc: add x0, x0, #0x160 0.00 : ffff80001011bfd0: mov x1, #0x1 // #1 0.00 : ffff80001011bfd4: str x21, [x29, #32] 0.00 : ffff80001011bfd8: asr w19, w19, #6 0.00 : ffff80001011bfdc: stp x23, x24, [x29, #48] 0.99 : ffff80001011bfe0: stp x25, x26, [x29, #64] : cpuidle_get_device(): : extern void cpuidle_disable_device(struct cpuidle_device *dev); : extern int cpuidle_play_dead(void); : : extern struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev); : static inline struct cpuidle_device *cpuidle_get_device(void) : {return __this_cpu_read(cpuidle_devices); } 0.00 : ffff80001011bfe4: adrp x23, ffff800011527000 : idle_should_enter_s2idle(): : : extern enum s2idle_states __read_mostly s2idle_state; : : static inline bool idle_should_enter_s2idle(void) : { : return unlikely(s2idle_state == S2IDLE_STATE_ENTER); 0.00 : ffff80001011bfe8: adrp x24, ffff80001190a000 <__per_cpu_offset+0x6d8> : cpuidle_get_device(): 0.00 : ffff80001011bfec: add x23, x23, #0xbf8 : idle_should_enter_s2idle(): 0.00 : ffff80001011bff0: add x24, x24, #0x25c : test_bit(): 0.00 : ffff80001011bff4: add x19, x0, w19, sxtw #3 0.00 : ffff80001011bff8: lsl x20, x1, x20 0.00 : ffff80001011bffc: adrp x21, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001011c000: b ffff80001011c01c : do_idle(): : * detected in the wakeup from idle path that the tick : * broadcast device expired for us, we don't want to go deep : * idle as we know that the IPI is going to arrive right away. : */ : if (cpu_idle_force_poll || tick_check_broadcast_expired()) { : tick_nohz_idle_restart_tick(); 0.00 : ffff80001011c004: bl ffff80001017c000 : cpu_idle_poll(); 0.00 : ffff80001011c008: bl ffff800010cd7b80 : } else { : cpuidle_idle_call(); : } : arch_cpu_idle_exit(); 0.00 : ffff80001011c00c: bl ffff80001011bf38 : get_current(): 0.00 : ffff80001011c010: mrs x0, sp_el0 : test_bit(): 0.00 : ffff80001011c014: ldr x0, [x0] : do_idle(): : while (!need_resched()) { 0.00 : ffff80001011c018: tbnz w0, #1, ffff80001011c0e0 : rmb(); 0.00 : ffff80001011c01c: dsb ld : arch_local_irq_disable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 15.63 : ffff80001011c020: mov x0, #0x60 // #96 0.00 : ffff80001011c024: msr daifset, #0x2 : test_bit(): 16.06 : ffff80001011c028: ldr x0, [x19] : do_idle(): : if (cpu_is_offline(cpu)) { 0.00 : ffff80001011c02c: tst x0, x20 0.00 : ffff80001011c030: b.eq ffff80001011c12c // b.none : arch_cpu_idle_enter(); 8.78 : ffff80001011c034: bl ffff80001011bf30 : if (cpu_idle_force_poll || tick_check_broadcast_expired()) { 0.00 : ffff80001011c038: ldr w0, [x21, #568] 0.00 : ffff80001011c03c: cbnz w0, ffff80001011c004 0.31 : ffff80001011c040: bl ffff80001017a460 3.49 : ffff80001011c044: cbnz w0, ffff80001011c004 : __my_cpu_offset(): 0.00 : ffff80001011c048: mrs x1, tpidr_el1 : cpuidle_get_device(): 2.58 : ffff80001011c04c: mov x0, x23 0.00 : ffff80001011c050: ldr x25, [x0, x1] : cpuidle_idle_call(): : struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 0.00 : ffff80001011c054: mov x0, x25 0.00 : ffff80001011c058: bl ffff800010a837c8 : get_current(): 0.31 : ffff80001011c05c: mrs x1, sp_el0 : test_bit(): 0.00 : ffff80001011c060: ldr x1, [x1] : cpuidle_idle_call(): 0.00 : ffff80001011c064: mov x26, x0 : if (need_resched()) { 0.00 : ffff80001011c068: tbnz w1, #1, ffff80001011c1c8 : if (cpuidle_not_available(drv, dev)) { 0.00 : ffff80001011c06c: mov x1, x25 0.00 : ffff80001011c070: bl ffff800010a82dc8 0.00 : ffff80001011c074: tst w0, #0xff 0.00 : ffff80001011c078: b.ne ffff80001011c13c // b.any : if (idle_should_enter_s2idle() || dev->forced_idle_latency_limit_ns) { 0.00 : ffff80001011c07c: ldr w0, [x24] 0.00 : ffff80001011c080: str x27, [x29, #80] 0.00 : ffff80001011c084: cmp w0, #0x1 0.00 : ffff80001011c088: b.eq ffff80001011c1ec // b.none 0.00 : ffff80001011c08c: ldr x27, [x25, #40] 0.00 : ffff80001011c090: cbz x27, ffff80001011c14c : tick_nohz_idle_stop_tick(); 0.00 : ffff80001011c094: bl ffff80001017bb98 : rcu_idle_enter(); 0.00 : ffff80001011c098: bl ffff800010160f40 : next_state = cpuidle_find_deepest_state(drv, dev, max_latency_ns); 0.00 : ffff80001011c09c: mov x2, x27 0.00 : ffff80001011c0a0: mov x1, x25 0.00 : ffff80001011c0a4: mov x0, x26 0.00 : ffff80001011c0a8: bl ffff800010a82f00 : call_cpuidle(drv, dev, next_state); 0.00 : ffff80001011c0ac: mov x1, x25 0.00 : ffff80001011c0b0: mov w2, w0 0.00 : ffff80001011c0b4: mov x0, x26 0.00 : ffff80001011c0b8: bl ffff80001011be90 0.00 : ffff80001011c0bc: ldr x27, [x29, #80] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001011c0c0: mrs x1, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001011c0c4: and w0, w1, #0x80 : cpuidle_idle_call(): : if (WARN_ON_ONCE(irqs_disabled())) 0.00 : ffff80001011c0c8: cbnz w0, ffff80001011c19c : rcu_idle_exit(); 0.00 : ffff80001011c0cc: bl ffff800010161098 : do_idle(): : arch_cpu_idle_exit(); 11.56 : ffff80001011c0d0: bl ffff80001011bf38 : get_current(): 0.00 : ffff80001011c0d4: mrs x0, sp_el0 : test_bit(): 0.10 : ffff80001011c0d8: ldr x0, [x0] : do_idle(): : while (!need_resched()) { 0.00 : ffff80001011c0dc: tbz w0, #1, ffff80001011c01c 0.91 : ffff80001011c0e0: ldr x19, [x29, #16] 0.00 : ffff80001011c0e4: ldr x21, [x29, #32] 0.00 : ffff80001011c0e8: ldp x23, x24, [x29, #48] 0.10 : ffff80001011c0ec: ldp x25, x26, [x29, #64] : get_current(): 0.20 : ffff80001011c0f0: mrs x0, sp_el0 : set_preempt_need_resched(): : task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \ : } while (0) : : static inline void set_preempt_need_resched(void) : { : current_thread_info()->preempt.need_resched = 0; 0.00 : ffff80001011c0f4: str wzr, [x0, #20] : do_idle(): : * : * This is required because for polling idle loops we will not have had : * an IPI to fold the state for us. : */ : preempt_set_need_resched(); : tick_nohz_idle_exit(); 0.00 : ffff80001011c0f8: bl ffff80001017c040 : /* : * We promise to call sched_ttwu_pending() and reschedule if : * need_resched() is set while polling is set. That means that clearing : * polling needs to be visible before doing these things. : */ : smp_mb__after_atomic(); 1.13 : ffff80001011c0fc: dmb ish : : sched_ttwu_pending(); 7.89 : ffff80001011c100: bl ffff800010117318 : schedule_idle(); : : if (unlikely(klp_patch_pending(current))) : klp_update_patch_state(current); : } 0.00 : ffff80001011c104: add x22, x22, #0x908 : schedule_idle(); 0.00 : ffff80001011c108: bl ffff800010cd2ed8 : } 2.59 : ffff80001011c10c: ldr x1, [x29, #104] 8.48 : ffff80001011c110: ldr x0, [x22] 0.00 : ffff80001011c114: eor x0, x1, x0 0.00 : ffff80001011c118: cbnz x0, ffff80001011c230 0.00 : ffff80001011c11c: ldr x20, [sp, #24] 1.86 : ffff80001011c120: ldr x22, [sp, #40] 0.00 : ffff80001011c124: ldp x29, x30, [sp], #112 0.00 : ffff80001011c128: ret : tick_nohz_idle_stop_tick(); 0.00 : ffff80001011c12c: bl ffff80001017bb98 : cpuhp_report_idle_dead(); 0.00 : ffff80001011c130: bl ffff8000100e89d8 : arch_cpu_idle_dead(); 0.00 : ffff80001011c134: bl ffff800010087580 0.00 : ffff80001011c138: b ffff80001011c034 : cpuidle_idle_call(): : tick_nohz_idle_stop_tick(); 0.00 : ffff80001011c13c: bl ffff80001017bb98 : rcu_idle_enter(); 0.50 : ffff80001011c140: bl ffff800010160f40 : default_idle_call(); 0.00 : ffff80001011c144: bl ffff800010cd7bf8 0.91 : ffff80001011c148: b ffff80001011c0c0 : bool stop_tick = true; 0.00 : ffff80001011c14c: mov w0, #0x1 // #1 : next_state = cpuidle_select(drv, dev, &stop_tick); 0.00 : ffff80001011c150: add x2, x29, #0x67 : bool stop_tick = true; 0.00 : ffff80001011c154: strb w0, [x29, #103] : next_state = cpuidle_select(drv, dev, &stop_tick); 0.00 : ffff80001011c158: mov x1, x25 0.00 : ffff80001011c15c: mov x0, x26 0.00 : ffff80001011c160: bl ffff800010a833a8 0.00 : ffff80001011c164: mov w27, w0 : if (stop_tick || tick_nohz_tick_stopped()) 0.00 : ffff80001011c168: ldrb w0, [x29, #103] 0.00 : ffff80001011c16c: cbz w0, ffff80001011c1b4 : tick_nohz_idle_stop_tick(); 0.00 : ffff80001011c170: bl ffff80001017bb98 : rcu_idle_enter(); 0.00 : ffff80001011c174: bl ffff800010160f40 : entered_state = call_cpuidle(drv, dev, next_state); 0.00 : ffff80001011c178: mov w2, w27 0.00 : ffff80001011c17c: mov x1, x25 0.00 : ffff80001011c180: mov x0, x26 0.00 : ffff80001011c184: bl ffff80001011be90 : cpuidle_reflect(dev, entered_state); 0.00 : ffff80001011c188: mov w1, w0 0.00 : ffff80001011c18c: mov x0, x25 0.00 : ffff80001011c190: bl ffff800010a83418 0.00 : ffff80001011c194: ldr x27, [x29, #80] 0.00 : ffff80001011c198: b ffff80001011c0c0 : if (WARN_ON_ONCE(irqs_disabled())) 0.00 : ffff80001011c19c: brk #0x800 : arch_local_irq_enable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001011c1a0: mov x0, #0xe0 // #224 0.00 : ffff80001011c1a4: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011c1a8: nop : cpuidle_idle_call(): : rcu_idle_exit(); 0.00 : ffff80001011c1ac: bl ffff800010161098 0.00 : ffff80001011c1b0: b ffff80001011c0d0 : if (stop_tick || tick_nohz_tick_stopped()) 0.00 : ffff80001011c1b4: bl ffff80001017bb50 0.00 : ffff80001011c1b8: tst w0, #0xff 0.00 : ffff80001011c1bc: b.ne ffff80001011c170 // b.any : tick_nohz_idle_retain_tick(); 0.00 : ffff80001011c1c0: bl ffff80001017bdd0 0.00 : ffff80001011c1c4: b ffff80001011c174 : arch_local_irq_enable(): 0.00 : ffff80001011c1c8: mov x0, #0xe0 // #224 0.00 : ffff80001011c1cc: msr daifclr, #0x2 : arch_static_branch(): 0.00 : ffff80001011c1d0: nop 0.00 : ffff80001011c1d4: b ffff80001011c00c : arch_local_irq_enable(): : pmr_sync(); 0.00 : ffff80001011c1d8: dsb sy 0.20 : ffff80001011c1dc: b ffff80001011c00c 0.00 : ffff80001011c1e0: dsb sy : cpuidle_idle_call(): : rcu_idle_exit(); 0.00 : ffff80001011c1e4: bl ffff800010161098 0.00 : ffff80001011c1e8: b ffff80001011c0d0 : rcu_idle_enter(); 0.00 : ffff80001011c1ec: bl ffff800010160f40 : entered_state = cpuidle_enter_s2idle(drv, dev); 0.00 : ffff80001011c1f0: mov x1, x25 0.00 : ffff80001011c1f4: mov x0, x26 0.00 : ffff80001011c1f8: bl ffff800010a82f60 : if (entered_state > 0) { 0.00 : ffff80001011c1fc: cmp w0, #0x0 0.00 : ffff80001011c200: b.le ffff80001011c218 : arch_local_irq_enable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001011c204: mov x0, #0xe0 // #224 0.00 : ffff80001011c208: msr daifclr, #0x2 : arch_static_branch(): 0.00 : ffff80001011c20c: nop 0.00 : ffff80001011c210: ldr x27, [x29, #80] 0.00 : ffff80001011c214: b ffff80001011c0c0 : cpuidle_idle_call(): : max_latency_ns = U64_MAX; 0.00 : ffff80001011c218: mov x27, #0xffffffffffffffff // #-1 : rcu_idle_exit(); 0.00 : ffff80001011c21c: bl ffff800010161098 0.00 : ffff80001011c220: b ffff80001011c094 : arch_local_irq_enable(): : pmr_sync(); 0.00 : ffff80001011c224: dsb sy 0.00 : ffff80001011c228: ldr x27, [x29, #80] 0.00 : ffff80001011c22c: b ffff80001011c0c0 0.00 : ffff80001011c230: str x19, [x29, #16] 0.00 : ffff80001011c234: str x21, [x29, #32] 0.00 : ffff80001011c238: stp x23, x24, [x29, #48] 0.00 : ffff80001011c23c: stp x25, x26, [x29, #64] 0.00 : ffff80001011c240: str x27, [x29, #80] : do_idle(): : } 0.00 : ffff80001011c244: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (886 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010115f40 : : ttwu_do_activate(): : } : #endif : } : : static void : ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 0.00 : ffff800010115f40: stp x29, x30, [sp, #-48]! 0.00 : ffff800010115f44: mov x29, sp 29.58 : ffff800010115f48: stp x19, x20, [sp, #16] 0.00 : ffff800010115f4c: mov x19, x0 0.00 : ffff800010115f50: str x21, [sp, #32] 0.00 : ffff800010115f54: mov x20, x1 0.00 : ffff800010115f58: mov w21, w2 : int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; : : lockdep_assert_held(&rq->lock); : : #ifdef CONFIG_SMP : if (p->sched_contributes_to_load) 0.00 : ffff800010115f5c: ldrb w0, [x1, #1060] 0.00 : ffff800010115f60: tbz w0, #1, ffff800010115f70 : rq->nr_uninterruptible--; 0.00 : ffff800010115f64: ldr x0, [x19, #2344] 0.00 : ffff800010115f68: sub x0, x0, #0x1 0.00 : ffff800010115f6c: str x0, [x19, #2344] : : if (wake_flags & WF_MIGRATED) : en_flags |= ENQUEUE_MIGRATED; 0.23 : ffff800010115f70: tst x21, #0x4 0.00 : ffff800010115f74: mov w0, #0x49 // #73 : #endif : : activate_task(rq, p, en_flags); 0.00 : ffff800010115f78: mov x1, x20 : en_flags |= ENQUEUE_MIGRATED; 0.00 : ffff800010115f7c: mov w2, #0x9 // #9 : activate_task(rq, p, en_flags); 24.76 : ffff800010115f80: csel w2, w2, w0, eq // eq = none 0.00 : ffff800010115f84: mov x0, x19 0.00 : ffff800010115f88: bl ffff800010115000 : ttwu_do_wakeup(rq, p, wake_flags, rf); 6.53 : ffff800010115f8c: mov w2, w21 7.61 : ffff800010115f90: mov x1, x20 1.47 : ffff800010115f94: mov x0, x19 0.21 : ffff800010115f98: bl ffff800010115ec0 : } 1.24 : ffff800010115f9c: ldp x19, x20, [sp, #16] 25.63 : ffff800010115fa0: ldr x21, [sp, #32] 2.74 : ffff800010115fa4: ldp x29, x30, [sp], #48 0.00 : ffff800010115fa8: ret Percent | Source code & Disassembly of vmlinux for cycles (856 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107205f0 : : arm_smmu_iotlb_sync(): : arm_smmu_tlb_inv_context(smmu_domain); : } : : static void arm_smmu_iotlb_sync(struct iommu_domain *domain, : struct iommu_iotlb_gather *gather) : { 40.58 : ffff8000107205f0: stp x29, x30, [sp, #-16]! : struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); : : arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start, 0.00 : ffff8000107205f4: sub x4, x0, #0x88 0.00 : ffff8000107205f8: mov w3, #0x1 // #1 : { 0.00 : ffff8000107205fc: mov x29, sp : arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start, 25.01 : ffff800010720600: ldr x5, [x1] 1.14 : ffff800010720604: ldr x2, [x1, #16] 0.00 : ffff800010720608: ldr x1, [x1, #8] 0.00 : ffff80001072060c: mov x0, x5 0.00 : ffff800010720610: sub x1, x1, x5 0.00 : ffff800010720614: bl ffff800010720478 : gather->pgsize, true, smmu_domain); : } 33.27 : ffff800010720618: ldp x29, x30, [sp], #16 0.00 : ffff80001072061c: ret Percent | Source code & Disassembly of vmlinux for cycles (860 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017c110 : : tick_irq_enter(): : : /* : * Called from irq_enter to notify about the possible interruption of idle() : */ : void tick_irq_enter(void) : { 0.11 : ffff80001017c110: stp x29, x30, [sp, #-48]! 0.00 : ffff80001017c114: mov x29, sp 0.12 : ffff80001017c118: stp x19, x20, [sp, #16] : tick_check_oneshot_broadcast_this_cpu(); 0.00 : ffff80001017c11c: bl ffff80001017a4a0 : tick_nohz_irq_enter(): : struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); 1.77 : ffff80001017c120: adrp x20, ffff800011525000 0.00 : ffff80001017c124: add x0, x20, #0xed0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001017c128: mrs x19, tpidr_el1 : tick_nohz_irq_enter(): 0.00 : ffff80001017c12c: add x19, x0, x19 : if (!ts->idle_active && !ts->tick_stopped) 0.00 : ffff80001017c130: ldrb w0, [x19, #76] 0.00 : ffff80001017c134: tst w0, #0x6 0.00 : ffff80001017c138: b.ne ffff80001017c148 // b.any : tick_irq_enter(): : tick_nohz_irq_enter(); : } 13.60 : ffff80001017c13c: ldp x19, x20, [sp, #16] 0.00 : ffff80001017c140: ldp x29, x30, [sp], #48 0.00 : ffff80001017c144: ret 0.71 : ffff80001017c148: str x21, [x29, #32] : tick_nohz_irq_enter(): : now = ktime_get(); 0.00 : ffff80001017c14c: bl ffff80001016c938 0.00 : ffff80001017c150: mov x21, x0 : if (ts->idle_active) 0.00 : ffff80001017c154: ldrb w0, [x19, #76] 0.00 : ffff80001017c158: tbnz w0, #2, ffff80001017c170 : if (ts->tick_stopped) 0.00 : ffff80001017c15c: tbnz w0, #1, ffff80001017c184 0.00 : ffff80001017c160: ldr x21, [x29, #32] : tick_irq_enter(): : } 0.00 : ffff80001017c164: ldp x19, x20, [sp, #16] 0.00 : ffff80001017c168: ldp x29, x30, [sp], #48 0.00 : ffff80001017c16c: ret : tick_nohz_irq_enter(): : tick_nohz_stop_idle(ts, now); 0.12 : ffff80001017c170: mov x0, x19 0.00 : ffff80001017c174: mov x1, x21 0.00 : ffff80001017c178: bl ffff80001017b6c0 0.12 : ffff80001017c17c: ldrb w0, [x19, #76] : if (ts->tick_stopped) 0.00 : ffff80001017c180: tbz w0, #1, ffff80001017c160 : tick_nohz_update_jiffies(): : __this_cpu_write(tick_cpu_sched.idle_waketime, now); 1.09 : ffff80001017c184: add x20, x20, #0xed0 : __my_cpu_offset(): 0.79 : ffff80001017c188: mrs x0, tpidr_el1 : tick_nohz_update_jiffies(): 0.00 : ffff80001017c18c: add x20, x20, #0x80 0.00 : ffff80001017c190: str x21, [x20, x0] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.48 : ffff80001017c194: mrs x19, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 1.06 : ffff80001017c198: and w0, w19, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff80001017c19c: cbnz w0, ffff80001017c1a8 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001017c1a0: mov x0, #0x60 // #96 0.00 : ffff80001017c1a4: msr daifset, #0x2 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001017c1a8: adrp x0, ffff800011b22000 : tick_do_update_jiffies64(): : if (delta < tick_period) 0.00 : ffff80001017c1ac: adrp x1, ffff800011b22000 : __read_once_size(): 0.12 : ffff80001017c1b0: ldr x0, [x0, #1328] : tick_do_update_jiffies64(): 0.12 : ffff80001017c1b4: ldr x1, [x1, #968] : delta = ktime_sub(now, READ_ONCE(last_jiffies_update)); 0.00 : ffff80001017c1b8: sub x0, x21, x0 : if (delta < tick_period) 0.00 : ffff80001017c1bc: cmp x0, x1 0.00 : ffff80001017c1c0: b.lt ffff80001017c1cc // b.tstop 0.00 : ffff80001017c1c4: mov x0, x21 0.00 : ffff80001017c1c8: bl ffff80001017b488 : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.68 : ffff80001017c1cc: msr daif, x19 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 14.44 : ffff80001017c1d0: nop 0.00 : ffff80001017c1d4: ldr x21, [x29, #32] : tick_irq_enter(): : } 0.00 : ffff80001017c1d8: b ffff80001017c13c : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.47 : ffff80001017c1dc: dsb sy 64.22 : ffff80001017c1e0: ldr x21, [x29, #32] 0.00 : ffff80001017c1e4: b ffff80001017c13c Percent | Source code & Disassembly of vmlinux for cycles (748 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010082a40 : : el1_irq(): : kernel_exit 1 : ENDPROC(el1_sync) : : .align 6 : el1_irq: : kernel_entry 1 0.00 : ffff800010082a40: stp x0, x1, [sp] 0.00 : ffff800010082a44: stp x2, x3, [sp, #16] 0.00 : ffff800010082a48: stp x4, x5, [sp, #32] 0.00 : ffff800010082a4c: stp x6, x7, [sp, #48] 0.00 : ffff800010082a50: stp x8, x9, [sp, #64] 0.00 : ffff800010082a54: stp x10, x11, [sp, #80] 0.00 : ffff800010082a58: stp x12, x13, [sp, #96] 0.00 : ffff800010082a5c: stp x14, x15, [sp, #112] 0.00 : ffff800010082a60: stp x16, x17, [sp, #128] 0.00 : ffff800010082a64: stp x18, x19, [sp, #144] 0.00 : ffff800010082a68: stp x20, x21, [sp, #160] 0.00 : ffff800010082a6c: stp x22, x23, [sp, #176] 0.00 : ffff800010082a70: stp x24, x25, [sp, #192] 0.00 : ffff800010082a74: stp x26, x27, [sp, #208] 0.00 : ffff800010082a78: stp x28, x29, [sp, #224] 0.00 : ffff800010082a7c: add x21, sp, #0x140 0.00 : ffff800010082a80: mrs x28, sp_el0 0.00 : ffff800010082a84: ldr x20, [x28, #8] 0.00 : ffff800010082a88: str x20, [sp, #288] 0.00 : ffff800010082a8c: mov x20, #0xffffffffffff // #281474976710655 0.00 : ffff800010082a90: str x20, [x28, #8] 0.00 : ffff800010082a94: mrs x22, elr_el1 0.00 : ffff800010082a98: mrs x23, spsr_el1 0.00 : ffff800010082a9c: stp x30, x21, [sp, #240] 0.00 : ffff800010082aa0: stp x29, x22, [sp, #304] 0.00 : ffff800010082aa4: add x29, sp, #0x130 0.00 : ffff800010082aa8: stp x22, x23, [sp, #256] 0.00 : ffff800010082aac: nop 0.00 : ffff800010082ab0: nop 0.00 : ffff800010082ab4: nop 0.00 : ffff800010082ab8: nop : gic_prio_irq_setup pmr=x20, tmp=x1 : enable_da_f 0.00 : ffff800010082abc: msr daifclr, #0xd 0.00 : ffff800010082ac0: mov x0, xzr : : #ifdef CONFIG_ARM64_PSEUDO_NMI : test_irqs_unmasked res=x0, pmr=x20 : cbz x0, 1f 0.00 : ffff800010082ac4: cbz x0, ffff800010082acc : bl asm_nmi_enter 0.00 : ffff800010082ac8: bl ffff800010084f30 : : #ifdef CONFIG_TRACE_IRQFLAGS : bl trace_hardirqs_off : #endif : : irq_handler 0.00 : ffff800010082acc: adrp x1, ffff8000112f2000 0.00 : ffff800010082ad0: ldr x1, [x1, #2072] 0.00 : ffff800010082ad4: mov x0, sp 0.00 : ffff800010082ad8: mov x19, sp 0.00 : ffff800010082adc: ldr x25, [x28, #32] 0.00 : ffff800010082ae0: eor x25, x25, x19 0.00 : ffff800010082ae4: and x25, x25, #0xffffffffffffc000 0.00 : ffff800010082ae8: cbnz x25, ffff800010082b08 0.00 : ffff800010082aec: adrp x25, ffff80001151d000 0.00 : ffff800010082af0: add x25, x25, #0x50 0.00 : ffff800010082af4: mrs x26, tpidr_el1 0.00 : ffff800010082af8: ldr x25, [x25, x26] 0.00 : ffff800010082afc: mov x26, #0x4000 // #16384 0.00 : ffff800010082b00: add x26, x25, x26 0.00 : ffff800010082b04: mov sp, x26 0.00 : ffff800010082b08: blr x1 32.39 : ffff800010082b0c: mov sp, x19 : : #ifdef CONFIG_PREEMPTION : ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count 6.77 : ffff800010082b10: ldr x24, [x28, #16] : * DA_F were cleared at start of handling. If anything is set in DAIF, : * we come back from an NMI, so skip preemption : */ : mrs x0, daif : orr x24, x24, x0 : alternative_else_nop_endif 23.49 : ffff800010082b14: nop 0.13 : ffff800010082b18: nop : cbnz x24, 1f // preempt count != 0 || NMI return path 0.00 : ffff800010082b1c: cbnz x24, ffff800010082b24 : bl arm64_preempt_schedule_irq // irq en/disable is done inside 0.27 : ffff800010082b20: bl ffff800010cd24c8 27.79 : ffff800010082b24: mov x0, xzr : * When using IRQ priority masking, we can get spurious interrupts while : * PMR is set to GIC_PRIO_IRQOFF. An NMI might also have occurred in a : * section with interrupts disabled. Skip tracing in those cases. : */ : test_irqs_unmasked res=x0, pmr=x20 : cbz x0, 1f 0.14 : ffff800010082b28: cbz x0, ffff800010082b30 : bl asm_nmi_exit 0.00 : ffff800010082b2c: bl ffff800010084fb0 : #endif : bl trace_hardirqs_on : 1: : #endif : : kernel_exit 1 9.02 : ffff800010082b30: msr daifset, #0xf 0.00 : ffff800010082b34: ldr x20, [sp, #288] 0.00 : ffff800010082b38: str x20, [x28, #8] 0.00 : ffff800010082b3c: nop 0.00 : ffff800010082b40: nop 0.00 : ffff800010082b44: nop 0.00 : ffff800010082b48: nop 0.00 : ffff800010082b4c: nop 0.00 : ffff800010082b50: ldp x21, x22, [sp, #256] 0.00 : ffff800010082b54: msr elr_el1, x21 0.00 : ffff800010082b58: msr spsr_el1, x22 0.00 : ffff800010082b5c: ldp x0, x1, [sp] 0.00 : ffff800010082b60: ldp x2, x3, [sp, #16] 0.00 : ffff800010082b64: ldp x4, x5, [sp, #32] 0.00 : ffff800010082b68: ldp x6, x7, [sp, #48] 0.00 : ffff800010082b6c: ldp x8, x9, [sp, #64] 0.00 : ffff800010082b70: ldp x10, x11, [sp, #80] 0.00 : ffff800010082b74: ldp x12, x13, [sp, #96] 0.00 : ffff800010082b78: ldp x14, x15, [sp, #112] 0.00 : ffff800010082b7c: ldp x16, x17, [sp, #128] 0.00 : ffff800010082b80: ldp x18, x19, [sp, #144] 0.00 : ffff800010082b84: ldp x20, x21, [sp, #160] 0.00 : ffff800010082b88: ldp x22, x23, [sp, #176] 0.00 : ffff800010082b8c: ldp x24, x25, [sp, #192] 0.00 : ffff800010082b90: ldp x26, x27, [sp, #208] 0.00 : ffff800010082b94: ldp x28, x29, [sp, #224] 0.00 : ffff800010082b98: ldr x30, [sp, #240] 0.00 : ffff800010082b9c: add sp, sp, #0x140 0.00 : ffff800010082ba0: eret 0.00 : ffff800010082ba4: dsb nsh 0.00 : ffff800010082ba8: isb Percent | Source code & Disassembly of vmlinux for cycles (660 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010456ab8 : : bio_put(): : * Description: : * Put a reference to a &struct bio, either one you have gotten with : * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it. : **/ : void bio_put(struct bio *bio) : { 0.15 : ffff800010456ab8: stp x29, x30, [sp, #-16]! 0.00 : ffff800010456abc: mov x29, sp : bio_flagged(): : atomic_set(&bio->__bi_cnt, count); : } : : static inline bool bio_flagged(struct bio *bio, unsigned int bit) : { : return (bio->bi_flags & (1U << bit)) != 0; 49.08 : ffff800010456ac0: ldrb w1, [x0, #21] : bio_put(): : if (!bio_flagged(bio, BIO_REFFED)) 0.00 : ffff800010456ac4: tbz w1, #0, ffff800010456b08 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010456ac8: ldr w1, [x0, #100] : bio_put(): : bio_free(bio); : else { : BIO_BUG_ON(!atomic_read(&bio->__bi_cnt)); 0.00 : ffff800010456acc: cbz w1, ffff800010456b14 : : /* : * last put frees it : */ : if (atomic_dec_and_test(&bio->__bi_cnt)) 0.00 : ffff800010456ad0: add x2, x0, #0x64 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010456ad4: b ffff800010456af8 0.00 : ffff800010456ad8: b ffff800010456af8 : __lse_atomic_sub_return(): : } : : ATOMIC_OP_SUB_RETURN(_relaxed, ) : ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC_OP_SUB_RETURN(_release, l, "memory") : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff800010456adc: mov w1, #0x1 // #1 0.00 : ffff800010456ae0: neg w1, w1 0.00 : ffff800010456ae4: ldaddal w1, w3, [x2] 0.00 : ffff800010456ae8: add w1, w1, w3 : bio_put(): 0.00 : ffff800010456aec: cbz w1, ffff800010456b08 : bio_free(bio); : } : } 0.00 : ffff800010456af0: ldp x29, x30, [sp], #16 0.00 : ffff800010456af4: ret : __ll_sc_atomic_sub_return(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010456af8: mov w2, #0x1 // #1 0.00 : ffff800010456afc: add x4, x0, #0x64 0.00 : ffff800010456b00: b ffff800010458b7c : bio_put(): : if (atomic_dec_and_test(&bio->__bi_cnt)) 0.00 : ffff800010456b04: cbnz w1, ffff800010456af0 : bio_free(bio); 0.76 : ffff800010456b08: bl ffff800010456a58 : } 50.01 : ffff800010456b0c: ldp x29, x30, [sp], #16 0.00 : ffff800010456b10: ret : BIO_BUG_ON(!atomic_read(&bio->__bi_cnt)); 0.00 : ffff800010456b14: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (710 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100ecc58 : : irq_enter(): : : /* : * Enter an interrupt context. : */ : void irq_enter(void) : { 0.00 : ffff8000100ecc58: stp x29, x30, [sp, #-32]! 0.00 : ffff8000100ecc5c: mov x29, sp 0.00 : ffff8000100ecc60: str x19, [sp, #16] : rcu_irq_enter(); 0.00 : ffff8000100ecc64: bl ffff800010161188 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 22.40 : ffff8000100ecc68: mrs x0, sp_el0 : is_idle_task(): : * : * Return: 1 if @p is an idle task. 0 otherwise. : */ : static inline bool is_idle_task(const struct task_struct *p) : { : return !!(p->flags & PF_IDLE); 9.68 : ffff8000100ecc6c: ldr w1, [x0, #44] : irq_enter(): : if (is_idle_task(current) && !in_interrupt()) { 0.52 : ffff8000100ecc70: tbz w1, #1, ffff8000100ecc80 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.62 : ffff8000100ecc74: ldr w1, [x0, #16] : irq_enter(): 0.00 : ffff8000100ecc78: tst w1, #0x1fff00 0.00 : ffff8000100ecc7c: b.eq ffff8000100ecca4 // b.none : get_current(): 6.90 : ffff8000100ecc80: mrs x19, sp_el0 : account_irq_enter_time(): : #endif : : static inline void account_irq_enter_time(struct task_struct *tsk) : { : vtime_account_irq_enter(tsk); : irqtime_account_irq(tsk); 1.83 : ffff8000100ecc84: mov x0, x19 0.00 : ffff8000100ecc88: bl ffff80001011b180 : __read_once_size(): 42.00 : ffff8000100ecc8c: ldr w0, [x19, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000100ecc90: add w0, w0, #0x10, lsl #12 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 12.34 : ffff8000100ecc94: str w0, [x19, #16] : irq_enter(): : tick_irq_enter(); : _local_bh_enable(); : } : : __irq_enter(); : } 0.14 : ffff8000100ecc98: ldr x19, [sp, #16] 0.00 : ffff8000100ecc9c: ldp x29, x30, [sp], #32 0.00 : ffff8000100ecca0: ret : __read_once_size(): : __READ_ONCE_SIZE; 1.06 : ffff8000100ecca4: ldr w1, [x0, #16] : __preempt_count_add(): 0.00 : ffff8000100ecca8: add w1, w1, #0x200 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.27 : ffff8000100eccac: str w1, [x0, #16] : irq_enter(): : tick_irq_enter(); 0.00 : ffff8000100eccb0: bl ffff80001017c110 : _local_bh_enable(); 0.25 : ffff8000100eccb4: bl ffff8000100ec468 <_local_bh_enable> 0.00 : ffff8000100eccb8: b ffff8000100ecc80 Percent | Source code & Disassembly of vmlinux for cycles (606 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010714278 : : iommu_dma_unmap_page(): : return dma_handle; : } : : static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, : size_t size, enum dma_data_direction dir, unsigned long attrs) : { 18.25 : ffff800010714278: stp x29, x30, [sp, #-64]! 0.16 : ffff80001071427c: mov x29, sp 22.99 : ffff800010714280: stp x19, x20, [sp, #16] 4.93 : ffff800010714284: mov x19, x0 6.47 : ffff800010714288: str x21, [sp, #32] 0.81 : ffff80001071428c: mov x20, x1 0.99 : ffff800010714290: mov x21, x2 : if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 0.00 : ffff800010714294: tbnz w4, #5, ffff8000107142a0 : dev_is_dma_coherent(): : #elif defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ : defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ : defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) : static inline bool dev_is_dma_coherent(struct device *dev) : { : return dev->dma_coherent; 2.93 : ffff800010714298: ldrb w1, [x0, #760] : iommu_dma_sync_single_for_cpu(): : if (dev_is_dma_coherent(dev)) 1.33 : ffff80001071429c: tbz w1, #4, ffff8000107142c0 : iommu_dma_unmap_page(): : iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir); : __iommu_dma_unmap(dev, dma_handle, size); 30.51 : ffff8000107142a0: mov x2, x21 0.00 : ffff8000107142a4: mov x1, x20 0.00 : ffff8000107142a8: mov x0, x19 0.16 : ffff8000107142ac: bl ffff800010713aa0 <__iommu_dma_unmap> : } 0.16 : ffff8000107142b0: ldp x19, x20, [sp, #16] 9.97 : ffff8000107142b4: ldr x21, [sp, #32] 0.33 : ffff8000107142b8: ldp x29, x30, [sp], #64 0.00 : ffff8000107142bc: ret 0.00 : ffff8000107142c0: str w3, [x29, #60] : iommu_dma_sync_single_for_cpu(): : phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); 0.00 : ffff8000107142c4: bl ffff800010712818 0.00 : ffff8000107142c8: mov x1, x20 0.00 : ffff8000107142cc: bl ffff800010710658 : arch_sync_dma_for_cpu(phys, size, dir); 0.00 : ffff8000107142d0: ldr w3, [x29, #60] 0.00 : ffff8000107142d4: mov x1, x21 0.00 : ffff8000107142d8: mov w2, w3 0.00 : ffff8000107142dc: bl ffff8000100a1c70 : iommu_dma_unmap_page(): : __iommu_dma_unmap(dev, dma_handle, size); 0.00 : ffff8000107142e0: mov x2, x21 0.00 : ffff8000107142e4: mov x1, x20 0.00 : ffff8000107142e8: mov x0, x19 0.00 : ffff8000107142ec: bl ffff800010713aa0 <__iommu_dma_unmap> : } 0.00 : ffff8000107142f0: ldp x19, x20, [sp, #16] 0.00 : ffff8000107142f4: ldr x21, [sp, #32] 0.00 : ffff8000107142f8: ldp x29, x30, [sp], #64 0.00 : ffff8000107142fc: ret Percent | Source code & Disassembly of vmlinux for cycles (1588 samples, percent: local period) ----------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010097120 : : el0_svc_common(): : } : #else : static void cortex_a76_erratum_1463225_svc_handler(void) { } : #endif /* CONFIG_ARM64_ERRATUM_1463225 */ : : static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, 0.00 : ffff800010097120: stp x29, x30, [sp, #-48]! 0.00 : ffff800010097124: mov x29, sp 0.00 : ffff800010097128: stp x19, x20, [sp, #16] 0.00 : ffff80001009712c: mov x19, x0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010097130: mrs x0, sp_el0 : el0_svc_common(): : const syscall_fn_t syscall_table[]) : { : unsigned long flags = current_thread_info()->flags; 0.00 : ffff800010097134: ldr x20, [x0] : : regs->orig_x0 = regs->regs[0]; 0.00 : ffff800010097138: ldr x3, [x19] 0.00 : ffff80001009713c: str x3, [x19, #272] : regs->syscallno = scno; 0.00 : ffff800010097140: str w1, [x19, #280] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010097144: ldr x0, [x0] : cortex_a76_erratum_1463225_svc_handler(): : if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) 0.00 : ffff800010097148: tst w0, #0x200000 0.00 : ffff80001009714c: b.ne ffff800010097250 // b.any : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010097150: b ffff8000100971a4 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010097154: nop : local_daif_restore(): : * So we don't need additional synchronization here. : */ : gic_write_pmr(pmr); : } : : write_sysreg(flags, daif); 0.00 : ffff800010097158: msr daif, xzr : el0_svc_common(): : : cortex_a76_erratum_1463225_svc_handler(); : local_daif_restore(DAIF_PROCCTX); : user_exit(); : : if (has_syscall_work(flags)) { 0.00 : ffff80001009715c: ands x20, x20, #0x1f80 0.00 : ffff800010097160: b.ne ffff800010097220 // b.any : invoke_syscall(): : if (scno < sc_nr) { 0.00 : ffff800010097164: cmp w1, #0x1b6 0.00 : ffff800010097168: b.hi ffff8000100972b4 // b.pmore : syscall_fn = syscall_table[array_index_nospec(scno, sc_nr)]; 0.38 : ffff80001009716c: mov w0, w1 : array_index_mask_nospec(): : static inline unsigned long array_index_mask_nospec(unsigned long idx, : unsigned long sz) : { : unsigned long mask; : : asm volatile( 0.00 : ffff800010097170: cmp x0, #0x1b7 0.00 : ffff800010097174: ngc x0, xzr : " sbc %0, xzr, xzr\n" : : "=r" (mask) : : "r" (idx), "Ir" (sz) : : "cc"); : : csdb(); 0.00 : ffff800010097178: csdb : invoke_syscall(): 0.06 : ffff80001009717c: and w1, w1, w0 : __invoke_syscall(): : return syscall_fn(regs); 0.00 : ffff800010097180: mov x0, x19 0.00 : ffff800010097184: ldr x1, [x2, x1, lsl #3] 0.00 : ffff800010097188: blr x1 : invoke_syscall(): : regs->regs[0] = ret; 0.00 : ffff80001009718c: str x0, [x19] : el0_svc_common(): : /* : * The tracing status may have changed under our feet, so we have to : * check again. However, if we were tracing entry, then we always trace : * exit regardless, as the old entry assembly did. : */ : if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) { 0.00 : ffff800010097190: cbnz x20, ffff8000100971fc : local_daif_mask(): : asm volatile( 0.19 : ffff800010097194: msr daifset, #0xf : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff800010097198: b ffff8000100971c0 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff80001009719c: nop 0.00 : ffff8000100971a0: b ffff8000100971d4 : test_bit(): 0.00 : ffff8000100971a4: adrp x0, ffff800011af2000 0.00 : ffff8000100971a8: ldr x0, [x0, #1320] : local_daif_restore(): : if (system_uses_irq_prio_masking()) { 0.00 : ffff8000100971ac: tbz x0, #42, ffff800010097158 : gic_write_pmr(): : return read_sysreg_s(SYS_ICC_PMR_EL1); : } : : static inline void gic_write_pmr(u32 val) : { : write_sysreg_s(val, SYS_ICC_PMR_EL1); 0.00 : ffff8000100971b0: mov x0, #0xe0 // #224 0.00 : ffff8000100971b4: msr s3_0_c4_c6_0, x0 : arch_static_branch(): 0.00 : ffff8000100971b8: nop 0.00 : ffff8000100971bc: b ffff800010097158 : test_bit(): 0.00 : ffff8000100971c0: adrp x0, ffff800011af2000 0.00 : ffff8000100971c4: ldr x0, [x0, #1320] : local_daif_mask(): : if (system_uses_irq_prio_masking()) 0.00 : ffff8000100971c8: tbz x0, #42, ffff8000100971d4 : gic_write_pmr(): 0.00 : ffff8000100971cc: mov x0, #0xf0 // #240 0.00 : ffff8000100971d0: msr s3_0_c4_c6_0, x0 : get_current(): 0.00 : ffff8000100971d4: mrs x0, sp_el0 : has_syscall_work(): : return unlikely(flags & _TIF_SYSCALL_WORK); 0.00 : ffff8000100971d8: ldr x0, [x0] : el0_svc_common(): : local_daif_mask(); : flags = current_thread_info()->flags; : if (!has_syscall_work(flags)) { 0.00 : ffff8000100971dc: tst x0, #0x1f80 0.00 : ffff8000100971e0: b.ne ffff8000100971f0 // b.any : local_daif_restore(DAIF_PROCCTX); : } : : trace_exit: : syscall_trace_exit(regs); : } 0.00 : ffff8000100971e4: ldp x19, x20, [sp, #16] 0.00 : ffff8000100971e8: ldp x29, x30, [sp], #48 0.00 : ffff8000100971ec: ret : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff8000100971f0: b ffff8000100972dc : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff8000100971f4: nop : local_daif_restore(): : write_sysreg(flags, daif); 0.00 : ffff8000100971f8: msr daif, xzr : el0_svc_common(): : syscall_trace_exit(regs); 0.00 : ffff8000100971fc: mov x0, x19 0.00 : ffff800010097200: bl ffff80001008b3b0 : } 0.00 : ffff800010097204: ldp x19, x20, [sp, #16] 0.00 : ffff800010097208: ldp x29, x30, [sp], #48 0.00 : ffff80001009720c: ret : local_daif_restore(): : pmr_sync(); 0.00 : ffff800010097210: dsb sy : write_sysreg(flags, daif); 0.00 : ffff800010097214: msr daif, xzr : el0_svc_common(): : if (has_syscall_work(flags)) { 99.38 : ffff800010097218: ands x20, x20, #0x1f80 0.00 : ffff80001009721c: b.eq ffff800010097164 // b.none : if (scno == NO_SYSCALL) 0.00 : ffff800010097220: cmn w1, #0x1 0.00 : ffff800010097224: b.ne ffff800010097230 // b.any : regs->regs[0] = -ENOSYS; 0.00 : ffff800010097228: mov x0, #0xffffffffffffffda // #-38 0.00 : ffff80001009722c: str x0, [x19] 0.00 : ffff800010097230: str x2, [x29, #40] : scno = syscall_trace_enter(regs); 0.00 : ffff800010097234: mov x0, x19 0.00 : ffff800010097238: bl ffff80001008b238 0.00 : ffff80001009723c: mov w1, w0 : if (scno == NO_SYSCALL) 0.00 : ffff800010097240: cmn w0, #0x1 0.00 : ffff800010097244: ldr x2, [x29, #40] 0.00 : ffff800010097248: b.ne ffff800010097164 // b.any 0.00 : ffff80001009724c: b ffff8000100971fc 0.00 : ffff800010097250: str x2, [x29, #32] : cortex_a76_erratum_1463225_svc_handler(): : if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225))) 0.00 : ffff800010097254: mov w0, #0x2c // #44 0.00 : ffff800010097258: str w1, [x29, #40] 0.00 : ffff80001009725c: bl ffff8000100954f8 0.00 : ffff800010097260: ldr w1, [x29, #40] 0.00 : ffff800010097264: tst w0, #0xff 0.00 : ffff800010097268: ldr x2, [x29, #32] 0.00 : ffff80001009726c: b.eq ffff800010097150 // b.none : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff800010097270: mov x5, sp : cortex_a76_erratum_1463225_svc_handler(): : __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); 0.00 : ffff800010097274: mov w4, #0x1 // #1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010097278: mrs x3, tpidr_el1 : cortex_a76_erratum_1463225_svc_handler(): 0.00 : ffff80001009727c: adrp x0, ffff80001151e000 0.00 : ffff800010097280: add x0, x0, #0x6a0 0.00 : ffff800010097284: str w4, [x0, x3] : reg = read_sysreg(mdscr_el1); 0.00 : ffff800010097288: mrs x3, mdscr_el1 : val = reg | DBG_MDSCR_SS | DBG_MDSCR_KDE; 0.00 : ffff80001009728c: mov w4, #0x2001 // #8193 : write_sysreg(val, mdscr_el1); 0.00 : ffff800010097290: orr w4, w4, w3 0.00 : ffff800010097294: msr mdscr_el1, x4 : asm volatile("msr daifclr, #8"); 0.00 : ffff800010097298: msr daifclr, #0x8 : isb(); 0.00 : ffff80001009729c: isb : write_sysreg(reg, mdscr_el1); 0.00 : ffff8000100972a0: and x3, x3, #0xffffffff 0.00 : ffff8000100972a4: msr mdscr_el1, x3 : __my_cpu_offset(): 0.00 : ffff8000100972a8: mrs x3, tpidr_el1 : cortex_a76_erratum_1463225_svc_handler(): : __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); 0.00 : ffff8000100972ac: str wzr, [x0, x3] 0.00 : ffff8000100972b0: b ffff800010097150 : get_current(): 0.00 : ffff8000100972b4: mrs x0, sp_el0 : test_bit(): 0.00 : ffff8000100972b8: ldr x0, [x0] : do_ni_syscall(): : if (is_compat_task()) { 0.00 : ffff8000100972bc: tst w0, #0x400000 0.00 : ffff8000100972c0: b.eq ffff8000100972d4 // b.none : ret = compat_arm_syscall(regs, scno); 0.00 : ffff8000100972c4: mov x0, x19 0.00 : ffff8000100972c8: bl ffff8000100998b8 : if (ret != -ENOSYS) 0.00 : ffff8000100972cc: cmn x0, #0x26 0.00 : ffff8000100972d0: b.ne ffff80001009718c // b.any : return sys_ni_syscall(); 0.00 : ffff8000100972d4: bl ffff80001010d700 0.00 : ffff8000100972d8: b ffff80001009718c : test_bit(): 0.00 : ffff8000100972dc: adrp x0, ffff800011af2000 0.00 : ffff8000100972e0: ldr x0, [x0, #1320] : local_daif_restore(): : if (system_uses_irq_prio_masking()) { 0.00 : ffff8000100972e4: tbz x0, #42, ffff8000100971f8 : gic_write_pmr(): 0.00 : ffff8000100972e8: mov x0, #0xe0 // #224 0.00 : ffff8000100972ec: msr s3_0_c4_c6_0, x0 : arch_static_branch(): 0.00 : ffff8000100972f0: nop 0.00 : ffff8000100972f4: b ffff8000100971f8 : local_daif_restore(): : pmr_sync(); 0.00 : ffff8000100972f8: dsb sy : write_sysreg(flags, daif); 0.00 : ffff8000100972fc: msr daif, xzr 0.00 : ffff800010097300: b ffff8000100971fc Percent | Source code & Disassembly of vmlinux for cycles (623 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012a808 : : enqueue_task_rt(): : /* : * Adding/removing a task to/from a priority array: : */ : static void : enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) : { 21.27 : ffff80001012a808: stp x29, x30, [sp, #-80]! 0.00 : ffff80001012a80c: mov x29, sp 13.02 : ffff80001012a810: stp x19, x20, [sp, #16] 0.00 : ffff80001012a814: mov x19, x1 0.66 : ffff80001012a818: stp x21, x22, [sp, #32] : struct sched_rt_entity *rt_se = &p->rt; 0.00 : ffff80001012a81c: add x22, x1, #0x1c0 : { 0.65 : ffff80001012a820: stp x23, x24, [sp, #48] 0.00 : ffff80001012a824: mov x23, x0 0.32 : ffff80001012a828: str x25, [sp, #64] 0.00 : ffff80001012a82c: mov w25, w2 : : if (flags & ENQUEUE_WAKEUP) 0.00 : ffff80001012a830: tbz w25, #0, ffff80001012a838 : rt_se->timeout = 0; 0.64 : ffff80001012a834: str xzr, [x22, #16] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.29 : ffff80001012a838: ldr w2, [x19, #68] : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012a83c: adrp x21, ffff800011909000 0.83 : ffff80001012a840: add x21, x21, #0x928 0.00 : ffff80001012a844: adrp x20, ffff800011528000 : enqueue_rt_entity(): : dequeue_rt_stack(rt_se, flags); 0.00 : ffff80001012a848: mov w1, w25 : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012a84c: add x20, x20, #0xe80 : enqueue_rt_entity(): : dequeue_rt_stack(rt_se, flags); 0.00 : ffff80001012a850: mov x0, x22 : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012a854: ldr x24, [x21, w2, uxtw #3] 0.00 : ffff80001012a858: add x24, x20, x24 : enqueue_rt_entity(): : dequeue_rt_stack(rt_se, flags); 0.00 : ffff80001012a85c: bl ffff80001012a5a0 : for_each_sched_rt_entity(rt_se) 2.74 : ffff80001012a860: cbz x22, ffff80001012a9a0 : __read_once_size(): 1.12 : ffff80001012a864: ldr w1, [x19, #68] : move_entity(): : if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) 0.00 : ffff80001012a868: and w0, w25, #0x6 0.00 : ffff80001012a86c: cmp w0, #0x2 : rt_se_prio(): : return rt_task_of(rt_se)->prio; 0.50 : ffff80001012a870: ldr w2, [x19, #108] : rq_of_rt_se(): : return task_rq(p); 0.48 : ffff80001012a874: ldr x0, [x21, w1, uxtw #3] 0.00 : ffff80001012a878: add x20, x20, x0 : rt_rq_of_se(): : return &rq->rt; 0.00 : ffff80001012a87c: add x21, x20, #0x200 : move_entity(): : if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) 3.26 : ffff80001012a880: b.eq ffff80001012a8f0 // b.none : __enqueue_rt_entity(): : WARN_ON_ONCE(rt_se->on_list); 0.52 : ffff80001012a884: ldrh w0, [x22, #38] 0.00 : ffff80001012a888: cbnz w0, ffff80001012aaac : struct list_head *queue = array->queue + rt_se_prio(rt_se); 5.07 : ffff80001012a88c: add x0, x20, #0x210 0.00 : ffff80001012a890: sbfiz x3, x2, #4, #32 0.00 : ffff80001012a894: add x1, x0, x3 : if (flags & ENQUEUE_HEAD) 0.00 : ffff80001012a898: tbnz w25, #4, ffff80001012aa2c : list_add_tail(): : * Insert a new entry before the specified head. : * This is useful for implementing queues. : */ : static inline void list_add_tail(struct list_head *new, struct list_head *head) : { : __list_add(new, head->prev, head); 0.66 : ffff80001012a89c: ldr x0, [x1, #8] : __list_add(): : next->prev = new; 4.37 : ffff80001012a8a0: str x22, [x1, #8] : new->prev = prev; 0.00 : ffff80001012a8a4: stp x1, x0, [x19, #448] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012a8a8: str x22, [x0] : rt_se_prio(): : return rt_task_of(rt_se)->prio; 4.97 : ffff80001012a8ac: ldr w1, [x19, #108] : __set_bit(): : * If it's called on the same region of memory simultaneously, the effect : * may be that only one operation succeeds. : */ : static inline void __set_bit(int nr, volatile unsigned long *addr) : { : unsigned long mask = BIT_MASK(nr); 0.00 : ffff80001012a8b0: mov x3, #0x1 // #1 : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff80001012a8b4: cmp w1, #0x0 0.00 : ffff80001012a8b8: add w0, w1, #0x3f 0.00 : ffff80001012a8bc: csel w0, w0, w1, lt // lt = tstop : unsigned long mask = BIT_MASK(nr); 0.00 : ffff80001012a8c0: negs w2, w1 0.00 : ffff80001012a8c4: and w2, w2, #0x3f 0.00 : ffff80001012a8c8: and w1, w1, #0x3f : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff80001012a8cc: asr w0, w0, #6 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff80001012a8d0: csneg w1, w1, w2, mi // mi = first : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff80001012a8d4: sxtw x0, w0 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff80001012a8d8: lsl x1, x3, x1 : : *p |= mask; 0.80 : ffff80001012a8dc: ldr x2, [x21, x0, lsl #3] 0.32 : ffff80001012a8e0: orr x1, x2, x1 6.30 : ffff80001012a8e4: str x1, [x21, x0, lsl #3] : __enqueue_rt_entity(): : rt_se->on_list = 1; 0.00 : ffff80001012a8e8: strh w3, [x22, #38] 0.00 : ffff80001012a8ec: ldr w2, [x19, #108] : rt_se->on_rq = 1; 0.00 : ffff80001012a8f0: mov w0, #0x1 // #1 0.00 : ffff80001012a8f4: strh w0, [x22, #36] : rt_prio(): : : struct task_struct; : : static inline int rt_prio(int prio) : { : if (unlikely(prio < MAX_RT_PRIO)) 0.00 : ffff80001012a8f8: cmp w2, #0x63 0.00 : ffff80001012a8fc: b.le ffff80001012a904 : inc_rt_tasks(): : WARN_ON(!rt_prio(prio)); 0.00 : ffff80001012a900: brk #0x800 : rt_rq->rt_nr_running += rt_se_nr_running(rt_se); 1.13 : ffff80001012a904: ldr w1, [x21, #1616] : rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se); 4.80 : ffff80001012a908: ldr w0, [x21, #1620] : rt_rq->rt_nr_running += rt_se_nr_running(rt_se); 0.00 : ffff80001012a90c: add w1, w1, #0x1 3.82 : ffff80001012a910: str w1, [x21, #1616] : inc_rt_prio(): : if (prio < prev_prio) 0.00 : ffff80001012a914: ldr w3, [x21, #1624] : rt_se_rr_nr_running(): : return (tsk->policy == SCHED_RR) ? 1 : 0; 0.18 : ffff80001012a918: ldr w1, [x19, #728] 0.00 : ffff80001012a91c: cmp w1, #0x2 : inc_rt_tasks(): : rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se); 0.00 : ffff80001012a920: cinc w0, w0, eq // eq = none 0.00 : ffff80001012a924: str w0, [x21, #1620] : inc_rt_prio(): : if (prio < prev_prio) 0.00 : ffff80001012a928: cmp w3, w2 0.00 : ffff80001012a92c: b.le ffff80001012a93c : rt_rq->highest_prio.curr = prio; 0.81 : ffff80001012a930: str w2, [x21, #1624] : inc_rt_prio_smp(): : if (rq->online && prio < prev_prio) 0.00 : ffff80001012a934: ldr w0, [x20, #2572] 0.00 : ffff80001012a938: cbnz w0, ffff80001012aa40 : inc_rt_migration(): : rt_rq->rt_nr_total++; 0.00 : ffff80001012a93c: ldr x0, [x21, #1640] 0.00 : ffff80001012a940: add x0, x0, #0x1 0.00 : ffff80001012a944: str x0, [x21, #1640] : if (p->nr_cpus_allowed > 1) 0.00 : ffff80001012a948: ldr w0, [x19, #732] 0.00 : ffff80001012a94c: cmp w0, #0x1 0.00 : ffff80001012a950: b.le ffff80001012a960 : rt_rq->rt_nr_migratory++; 0.00 : ffff80001012a954: ldr x0, [x21, #1632] 0.00 : ffff80001012a958: add x0, x0, #0x1 0.00 : ffff80001012a95c: str x0, [x21, #1632] : update_rt_migration(rt_rq); 1.29 : ffff80001012a960: mov x0, x21 0.00 : ffff80001012a964: bl ffff80001012a490 : rt_bandwidth_enabled(): : #endif /* CONFIG_FAIR_GROUP_SCHED */ : }; : : static inline int rt_bandwidth_enabled(void) : { : return sysctl_sched_rt_runtime >= 0; 0.00 : ffff80001012a968: adrp x0, ffff800011920000 : start_rt_bandwidth(): : if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF) 0.00 : ffff80001012a96c: ldr w0, [x0, #912] 0.00 : ffff80001012a970: tbnz w0, #31, ffff80001012a9a0 0.00 : ffff80001012a974: adrp x21, ffff800011afc000 0.00 : ffff80001012a978: add x20, x21, #0x0 1.88 : ffff80001012a97c: ldr x0, [x20, #16] 0.00 : ffff80001012a980: cmn x0, #0x1 0.00 : ffff80001012a984: b.eq ffff80001012a9a0 // b.none : raw_spin_lock(&rt_b->rt_runtime_lock); 0.92 : ffff80001012a988: mov x0, x20 0.00 : ffff80001012a98c: bl ffff800010cd80f0 <_raw_spin_lock> : if (!rt_b->rt_period_active) { 1.27 : ffff80001012a990: ldr w0, [x20, #88] 0.00 : ffff80001012a994: cbz w0, ffff80001012aa6c : raw_spin_unlock(&rt_b->rt_runtime_lock); 0.17 : ffff80001012a998: add x0, x21, #0x0 0.00 : ffff80001012a99c: bl ffff800010cd7c38 <_raw_spin_unlock> : enqueue_rt_entity(): : enqueue_top_rt_rq(&rq->rt); 4.37 : ffff80001012a9a0: add x0, x24, #0x200 0.00 : ffff80001012a9a4: bl ffff8000101292d0 : enqueue_task_rt(): : : enqueue_rt_entity(rt_se, flags); : : if (!task_current(rq, p) && p->nr_cpus_allowed > 1) 6.20 : ffff80001012a9a8: ldr x0, [x23, #2352] 0.00 : ffff80001012a9ac: cmp x19, x0 0.00 : ffff80001012a9b0: b.eq ffff80001012aa14 // b.none 0.48 : ffff80001012a9b4: ldr w0, [x19, #732] 0.00 : ffff80001012a9b8: cmp w0, #0x1 0.00 : ffff80001012a9bc: b.le ffff80001012aa14 : enqueue_pushable_task(): : plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 0.00 : ffff80001012a9c0: add x20, x19, #0x378 0.00 : ffff80001012a9c4: add x21, x23, #0x878 0.00 : ffff80001012a9c8: mov x1, x21 0.00 : ffff80001012a9cc: mov x0, x20 0.00 : ffff80001012a9d0: bl ffff800010cc2748 : plist_node_init(): : * @node: &struct plist_node pointer : * @prio: initial node priority : */ : static inline void plist_node_init(struct plist_node *node, int prio) : { : node->prio = prio; 0.00 : ffff80001012a9d4: ldr w2, [x19, #108] : INIT_LIST_HEAD(&node->prio_list); 0.00 : ffff80001012a9d8: add x1, x19, #0x380 : node->prio = prio; 0.00 : ffff80001012a9dc: str w2, [x19, #888] : INIT_LIST_HEAD(&node->node_list); 0.00 : ffff80001012a9e0: add x2, x19, #0x390 : __write_once_size(): 0.00 : ffff80001012a9e4: str x1, [x19, #896] : enqueue_pushable_task(): : plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks); 0.00 : ffff80001012a9e8: mov x0, x20 : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff80001012a9ec: str x1, [x19, #904] : enqueue_pushable_task(): 0.00 : ffff80001012a9f0: mov x1, x21 : __write_once_size(): 0.00 : ffff80001012a9f4: str x2, [x19, #912] : INIT_LIST_HEAD(): 0.00 : ffff80001012a9f8: str x2, [x19, #920] : enqueue_pushable_task(): 0.00 : ffff80001012a9fc: bl ffff800010cc2670 : if (p->prio < rq->rt.highest_prio.next) 0.00 : ffff80001012aa00: ldr w0, [x19, #108] 0.00 : ffff80001012aa04: ldr w1, [x23, #2140] 0.00 : ffff80001012aa08: cmp w0, w1 0.00 : ffff80001012aa0c: b.ge ffff80001012aa14 // b.tcont : rq->rt.highest_prio.next = p->prio; 0.00 : ffff80001012aa10: str w0, [x23, #2140] : enqueue_task_rt(): : enqueue_pushable_task(rq, p); : } 0.50 : ffff80001012aa14: ldp x19, x20, [sp, #16] 0.15 : ffff80001012aa18: ldp x21, x22, [sp, #32] 0.63 : ffff80001012aa1c: ldp x23, x24, [sp, #48] 0.00 : ffff80001012aa20: ldr x25, [sp, #64] 0.18 : ffff80001012aa24: ldp x29, x30, [sp], #80 0.00 : ffff80001012aa28: ret : list_add(): : __list_add(new, head, head->next); 0.00 : ffff80001012aa2c: ldr x2, [x0, x3] : __list_add(): : next->prev = new; 0.00 : ffff80001012aa30: str x22, [x2, #8] : new->prev = prev; 0.00 : ffff80001012aa34: stp x2, x1, [x19, #448] : __write_once_size(): 0.00 : ffff80001012aa38: str x22, [x0, x3] 0.00 : ffff80001012aa3c: b ffff80001012a8ac : inc_rt_prio_smp(): : cpupri_set(&rq->rd->cpupri, rq->cpu, prio); 0.29 : ffff80001012aa40: ldr w1, [x20, #2568] 0.16 : ffff80001012aa44: ldr x0, [x20, #2464] 0.00 : ffff80001012aa48: add x0, x0, #0x120 0.00 : ffff80001012aa4c: bl ffff800010130ab0 : inc_rt_migration(): : rt_rq->rt_nr_total++; 0.81 : ffff80001012aa50: ldr x0, [x21, #1640] 0.00 : ffff80001012aa54: add x0, x0, #0x1 0.18 : ffff80001012aa58: str x0, [x21, #1640] : if (p->nr_cpus_allowed > 1) 0.00 : ffff80001012aa5c: ldr w0, [x19, #732] 0.00 : ffff80001012aa60: cmp w0, #0x1 0.00 : ffff80001012aa64: b.gt ffff80001012a954 0.00 : ffff80001012aa68: b ffff80001012a960 : hrtimer_forward_now(): : * the timer. : */ : static inline u64 hrtimer_forward_now(struct hrtimer *timer, : ktime_t interval) : { : return hrtimer_forward(timer, timer->base->get_time(), interval); 0.00 : ffff80001012aa6c: ldr x0, [x20, #72] : start_rt_bandwidth(): : rt_b->rt_period_active = 1; 0.00 : ffff80001012aa70: mov w1, #0x1 // #1 0.00 : ffff80001012aa74: str w1, [x20, #88] : hrtimer_forward_now(): 0.00 : ffff80001012aa78: add x22, x20, #0x18 0.00 : ffff80001012aa7c: ldr x0, [x0, #48] 0.00 : ffff80001012aa80: blr x0 0.00 : ffff80001012aa84: mov x1, x0 0.00 : ffff80001012aa88: mov x2, #0x0 // #0 0.00 : ffff80001012aa8c: mov x0, x22 0.00 : ffff80001012aa90: bl ffff80001016acf8 : hrtimer_start_expires(): : soft = hrtimer_get_softexpires(timer); 0.00 : ffff80001012aa94: ldp x2, x1, [x20, #48] : hrtimer_start_range_ns(timer, soft, delta, mode); 0.00 : ffff80001012aa98: mov w3, #0xa // #10 0.00 : ffff80001012aa9c: mov x0, x22 0.00 : ffff80001012aaa0: sub x2, x2, x1 0.00 : ffff80001012aaa4: bl ffff80001016b750 0.00 : ffff80001012aaa8: b ffff80001012a998 : __enqueue_rt_entity(): : WARN_ON_ONCE(rt_se->on_list); 0.00 : ffff80001012aaac: brk #0x800 0.00 : ffff80001012aab0: b ffff80001012a88c Percent | Source code & Disassembly of vmlinux for cycles (534 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011b140 : : sched_clock_cpu(): : generic_sched_clock_init(); : local_irq_enable(); : } : : u64 sched_clock_cpu(int cpu) : { 51.74 : ffff80001011b140: stp x29, x30, [sp, #-16]! 0.00 : ffff80001011b144: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011b148: b ffff80001011b158 : sched_clock_cpu(): : if (!static_branch_likely(&sched_clock_running)) : return 0; : : return sched_clock(); 48.26 : ffff80001011b14c: bl ffff80001017af20 : } 0.00 : ffff80001011b150: ldp x29, x30, [sp], #16 0.00 : ffff80001011b154: ret : return 0; 0.00 : ffff80001011b158: mov x0, #0x0 // #0 : } 0.00 : ffff80001011b15c: ldp x29, x30, [sp], #16 0.00 : ffff80001011b160: ret Percent | Source code & Disassembly of vmlinux for cycles (552 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016af18 <__hrtimer_next_event_base>: : __hrtimer_next_event_base(): : : static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base, : const struct hrtimer *exclude, : unsigned int active, : ktime_t expires_next) : { 4.93 : ffff80001016af18: stp x29, x30, [sp, #-64]! 0.00 : ffff80001016af1c: mov x29, sp 14.21 : ffff80001016af20: str x22, [sp, #40] 0.00 : ffff80001016af24: mov x22, x3 : __next_base(): : if (!*active) 0.00 : ffff80001016af28: cbz w2, ffff80001016aff4 <__hrtimer_next_event_base+0xdc> 2.76 : ffff80001016af2c: str x21, [x29, #32] 0.00 : ffff80001016af30: mov x21, x0 : idx = __ffs(*active); 0.00 : ffff80001016af34: mov w0, w2 0.72 : ffff80001016af38: stp x19, x20, [x29, #16] : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 0.00 : ffff80001016af3c: rbit x0, x0 1.47 : ffff80001016af40: stp x23, x24, [x29, #48] 0.00 : ffff80001016af44: clz x0, x0 : __next_base(): : *active &= ~(1U << idx); 0.00 : ffff80001016af48: mov w24, #0x1 // #1 : return &cpu_base->clock_base[idx]; 0.00 : ffff80001016af4c: add x4, x0, #0x1 3.26 : ffff80001016af50: mov x23, x1 : *active &= ~(1U << idx); 0.00 : ffff80001016af54: lsl w20, w24, w0 : idx = __ffs(*active); 0.00 : ffff80001016af58: mov w19, w0 : *active &= ~(1U << idx); 0.00 : ffff80001016af5c: bic w20, w2, w20 : __hrtimer_next_event_base(): : struct hrtimer_clock_base *base; : ktime_t expires; : : for_each_active_base(base, cpu_base, active) { 0.00 : ffff80001016af60: adds x4, x21, x4, lsl #6 0.00 : ffff80001016af64: b.ne ffff80001016af94 <__hrtimer_next_event_base+0x7c> // b.any 0.00 : ffff80001016af68: b ffff80001016afe8 <__hrtimer_next_event_base+0xd0> : __next_base(): : idx = __ffs(*active); 2.19 : ffff80001016af6c: mov w0, w20 : __ffs(): 0.00 : ffff80001016af70: rbit x0, x0 0.00 : ffff80001016af74: clz x0, x0 : __next_base(): : return &cpu_base->clock_base[idx]; 0.00 : ffff80001016af78: add x4, x0, #0x1 : if (!*active) 2.56 : ffff80001016af7c: cbz w20, ffff80001016afe8 <__hrtimer_next_event_base+0xd0> : *active &= ~(1U << idx); 0.19 : ffff80001016af80: lsl w2, w24, w0 : idx = __ffs(*active); 0.00 : ffff80001016af84: mov w19, w0 : *active &= ~(1U << idx); 0.00 : ffff80001016af88: bic w20, w20, w2 : __hrtimer_next_event_base(): : for_each_active_base(base, cpu_base, active) { 0.00 : ffff80001016af8c: adds x4, x21, x4, lsl #6 0.38 : ffff80001016af90: b.eq ffff80001016afe8 <__hrtimer_next_event_base+0xd0> // b.none : struct timerqueue_node *next; : struct hrtimer *timer; : : next = timerqueue_getnext(&base->active); 1.47 : ffff80001016af94: ldr x0, [x4, #40] : timer = container_of(next, struct hrtimer, node); : if (timer == exclude) { 0.00 : ffff80001016af98: cmp x23, x0 0.00 : ffff80001016af9c: b.eq ffff80001016b008 <__hrtimer_next_event_base+0xf0> // b.none : if (!next) : continue; : : timer = container_of(next, struct hrtimer, node); : } : expires = ktime_sub(hrtimer_get_expires(timer), base->offset); 20.90 : ffff80001016afa0: ubfiz x19, x19, #6, #7 5.60 : ffff80001016afa4: ldr x4, [x0, #24] 0.00 : ffff80001016afa8: add x19, x21, x19 24.07 : ffff80001016afac: ldr x2, [x19, #120] 0.00 : ffff80001016afb0: sub x4, x4, x2 : if (expires < expires_next) { 0.00 : ffff80001016afb4: cmp x4, x22 0.00 : ffff80001016afb8: b.ge ffff80001016af6c <__hrtimer_next_event_base+0x54> // b.tcont 0.00 : ffff80001016afbc: mov x22, x4 : expires_next = expires; : : /* Skip cpu_base update if a timer is being excluded. */ : if (exclude) 0.00 : ffff80001016afc0: cbnz x23, ffff80001016af6c <__hrtimer_next_event_base+0x54> : continue; : : if (timer->is_soft) 3.94 : ffff80001016afc4: ldrb w1, [x0, #58] 0.00 : ffff80001016afc8: cbz w1, ffff80001016b018 <__hrtimer_next_event_base+0x100> : cpu_base->softirq_next_timer = timer; 0.00 : ffff80001016afcc: str x0, [x21, #56] : __next_base(): : idx = __ffs(*active); 0.00 : ffff80001016afd0: mov w0, w20 : __ffs(): 0.00 : ffff80001016afd4: rbit x0, x0 0.00 : ffff80001016afd8: clz x0, x0 : __next_base(): : return &cpu_base->clock_base[idx]; 0.00 : ffff80001016afdc: add x4, x0, #0x1 : if (!*active) 0.00 : ffff80001016afe0: cbnz w20, ffff80001016af80 <__hrtimer_next_event_base+0x68> 0.00 : ffff80001016afe4: nop 0.00 : ffff80001016afe8: ldp x19, x20, [x29, #16] 0.00 : ffff80001016afec: ldr x21, [x29, #32] 0.00 : ffff80001016aff0: ldp x23, x24, [x29, #48] : __hrtimer_next_event_base(): : * the clock bases so the result might be negative. Fix it up : * to prevent a false positive in clockevents_program_event(). : */ : if (expires_next < 0) : expires_next = 0; : return expires_next; 2.51 : ffff80001016aff4: cmp x22, #0x0 : } 0.00 : ffff80001016aff8: csel x0, x22, xzr, ge // ge = tcont 3.90 : ffff80001016affc: ldr x22, [sp, #40] 4.93 : ffff80001016b000: ldp x29, x30, [sp], #64 0.00 : ffff80001016b004: ret : next = timerqueue_iterate_next(next); 0.00 : ffff80001016b008: mov x0, x23 0.00 : ffff80001016b00c: bl ffff800010cc8b38 : if (!next) 0.00 : ffff80001016b010: cbnz x0, ffff80001016afa0 <__hrtimer_next_event_base+0x88> 0.00 : ffff80001016b014: b ffff80001016af6c <__hrtimer_next_event_base+0x54> : cpu_base->next_timer = timer; 0.00 : ffff80001016b018: str x0, [x21, #40] 0.00 : ffff80001016b01c: b ffff80001016af6c <__hrtimer_next_event_base+0x54> Percent | Source code & Disassembly of vmlinux for cycles (508 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100a3110 : : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 18.23 : ffff8000100a3110: ldr x1, [x0] : flush_dcache_page(): : * it as dirty for later flushing when mapped in user space (if executable, : * see __sync_icache_dcache). : */ : void flush_dcache_page(struct page *page) : { : if (test_bit(PG_dcache_clean, &page->flags)) 0.00 : ffff8000100a3114: tst w1, #0x800 0.00 : ffff8000100a3118: b.ne ffff8000100a3120 // b.any : clear_bit(PG_dcache_clean, &page->flags); : } 81.77 : ffff8000100a311c: ret : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000100a3120: b ffff8000100a3134 0.00 : ffff8000100a3124: b ffff8000100a3134 : __lse_atomic64_andnot(): : " " #asm_op " %[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) 0.00 : ffff8000100a3128: mov x1, #0x800 // #2048 0.00 : ffff8000100a312c: stclr x1, [x0] : flush_dcache_page(): 0.00 : ffff8000100a3130: ret : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff8000100a3134: mov x1, #0x800 // #2048 0.00 : ffff8000100a3138: b ffff8000100a3298 : flush_dcache_page(): 0.00 : ffff8000100a313c: ret Percent | Source code & Disassembly of vmlinux for cycles (504 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010134fe0 : : update_irq_load_avg(): : * runnable_load_sum = load_sum : * : */ : : int update_irq_load_avg(struct rq *rq, u64 running) : { 9.91 : ffff800010134fe0: mov x8, x0 4.98 : ffff800010134fe4: stp x29, x30, [sp, #-16]! : topology_get_freq_scale(): : DECLARE_PER_CPU(unsigned long, freq_scale); : : static inline : unsigned long topology_get_freq_scale(int cpu) : { : return per_cpu(freq_scale, cpu); 0.00 : ffff800010134fe8: adrp x4, ffff800011909000 0.00 : ffff800010134fec: add x4, x4, #0x928 : update_irq_load_avg(): 0.00 : ffff800010134ff0: mov x29, sp : topology_get_freq_scale(): 6.38 : ffff800010134ff4: ldrsw x6, [x8, #2568] 0.00 : ffff800010134ff8: adrp x0, ffff800011527000 0.19 : ffff800010134ffc: add x0, x0, #0xa08 : topology_get_cpu_scale(): : return per_cpu(cpu_scale, cpu); 0.62 : ffff800010135000: adrp x2, ffff800011527000 0.37 : ffff800010135004: add x2, x2, #0xa00 : update_irq_load_avg(): : * We start to decay with normal context time and then we add the : * interrupt context time. : * We can safely remove running from rq->clock because : * rq->clock += delta with delta >= running : */ : ret = ___update_load_sum(rq->clock - running, &rq->avg_irq, 0.98 : ffff800010135008: ldr x5, [x8, #2400] : topology_get_freq_scale(): : return per_cpu(freq_scale, cpu); 4.48 : ffff80001013500c: ldr x4, [x4, x6, lsl #3] : ___update_load_sum(): : delta = now - sa->last_update_time; 0.00 : ffff800010135010: add x7, x8, #0xac0 1.99 : ffff800010135014: ldr x3, [x8, #2752] : update_irq_load_avg(): : running = cap_scale(running, arch_scale_freq_capacity(cpu_of(rq))); 3.98 : ffff800010135018: ldr x0, [x0, x4] : running = cap_scale(running, arch_scale_cpu_capacity(cpu_of(rq))); 5.68 : ffff80001013501c: ldr x2, [x2, x4] : running = cap_scale(running, arch_scale_freq_capacity(cpu_of(rq))); 0.21 : ffff800010135020: mul x1, x1, x0 0.00 : ffff800010135024: lsr x1, x1, #10 : running = cap_scale(running, arch_scale_cpu_capacity(cpu_of(rq))); 0.00 : ffff800010135028: mul x1, x1, x2 0.20 : ffff80001013502c: lsr x1, x1, #10 : ret = ___update_load_sum(rq->clock - running, &rq->avg_irq, 0.00 : ffff800010135030: sub x0, x5, x1 : ___update_load_sum(): : if ((s64)delta < 0) { 0.00 : ffff800010135034: subs x2, x0, x3 0.00 : ffff800010135038: b.mi ffff8000101351b8 // b.first : delta >>= 10; 0.00 : ffff80001013503c: lsr x0, x2, #10 : if (!delta) 3.62 : ffff800010135040: cbnz x0, ffff800010135128 : if ((s64)delta < 0) { 0.00 : ffff800010135044: subs x1, x5, x3 0.00 : ffff800010135048: b.mi ffff8000101351f4 // b.first : delta >>= 10; 0.00 : ffff80001013504c: lsr x0, x1, #10 : if (!delta) 0.00 : ffff800010135050: mov w9, #0x0 // #0 0.00 : ffff800010135054: cbz x0, ffff80001013511c : sa->last_update_time += delta << 10; 3.47 : ffff800010135058: and x1, x1, #0xfffffffffffffc00 0.00 : ffff80001013505c: add x1, x1, x3 0.43 : ffff800010135060: str x1, [x8, #2752] : accumulate_sum(): : delta += sa->period_contrib; 2.05 : ffff800010135064: ldr w4, [x7, #28] 3.07 : ffff800010135068: ldr x10, [x7, #8] 0.00 : ffff80001013506c: add x2, x0, w4, uxtw : periods = delta / 1024; /* A period is 1024us (~1ms) */ 0.00 : ffff800010135070: lsr x14, x2, #10 : if (periods) { 0.00 : ffff800010135074: cbz x14, ffff8000101351c4 : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.20 : ffff800010135078: cmp x14, #0x7e0 0.00 : ffff80001013507c: b.hi ffff800010135210 // b.pmore : if (unlikely(local_n >= LOAD_AVG_PERIOD)) { 1.25 : ffff800010135080: cmp x14, #0x1f 0.00 : ffff800010135084: ldr w0, [x7, #24] 0.00 : ffff800010135088: ldr x3, [x7, #16] 0.00 : ffff80001013508c: b.hi ffff800010135270 // b.pmore : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010135090: adrp x1, ffff800010d06000 0.00 : ffff800010135094: add x1, x1, #0x450 : mul_u64_u32_shr(): : #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) : : #ifndef mul_u64_u32_shr : static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) : { : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff800010135098: ldr w12, [x1, w14, uxtw #2] 0.00 : ffff80001013509c: mul x5, x10, x12 0.00 : ffff8000101350a0: mul x1, x3, x12 0.00 : ffff8000101350a4: umulh x10, x10, x12 0.21 : ffff8000101350a8: umulh x3, x3, x12 0.00 : ffff8000101350ac: mul x0, x0, x12 0.00 : ffff8000101350b0: extr x10, x10, x5, #32 0.00 : ffff8000101350b4: extr x11, x3, x1, #32 0.00 : ffff8000101350b8: lsr x12, x0, #32 : accumulate_sum(): : delta %= 1024; 0.00 : ffff8000101350bc: and x2, x2, #0x3ff : contrib = __accumulate_pelt_segments(periods, 0.00 : ffff8000101350c0: mov w1, #0x400 // #1024 0.00 : ffff8000101350c4: mov w13, w2 0.00 : ffff8000101350c8: sub w1, w1, w4 0.00 : ffff8000101350cc: mov x0, x14 0.00 : ffff8000101350d0: bl ffff8000101346b8 <__accumulate_pelt_segments> : ___update_load_sum(): : if (!accumulate_sum(delta, sa, load, runnable, running)) 0.00 : ffff8000101350d4: cmp w14, #0x0 : accumulate_sum(): : sa->load_sum += load * contrib; 0.00 : ffff8000101350d8: mov w1, w0 0.00 : ffff8000101350dc: cinc w9, w9, ne // ne = any 0.00 : ffff8000101350e0: add x10, x1, x10 : sa->util_sum += contrib << SCHED_CAPACITY_SHIFT; 0.00 : ffff8000101350e4: add w0, w12, w0, lsl #10 : sa->runnable_load_sum += runnable * contrib; 0.00 : ffff8000101350e8: add x1, x1, x11 0.00 : ffff8000101350ec: stp x10, x1, [x7, #8] : sa->period_contrib = delta; 0.22 : ffff8000101350f0: stp w0, w13, [x7, #24] : update_irq_load_avg(): : ret += ___update_load_sum(rq->clock, &rq->avg_irq, : 1, : 1, : 1); : : if (ret) { 1.60 : ffff8000101350f4: cbz w9, ffff80001013511c : ___update_load_avg(): : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff8000101350f8: ldp w1, w0, [x7, #24] 0.00 : ffff8000101350fc: mov w2, #0xb67e // #46718 0.00 : ffff800010135100: add w4, w0, w2 : div_u64_rem(): : return dividend / divisor; 2.84 : ffff800010135104: ldp x3, x2, [x7, #8] : ___update_load_avg(): : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 0.00 : ffff800010135108: udiv w0, w1, w4 : div_u64_rem(): 0.00 : ffff80001013510c: udiv x3, x3, x4 0.00 : ffff800010135110: udiv x1, x2, x4 : ___update_load_avg(): : sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider); 0.22 : ffff800010135114: stp x3, x1, [x7, #32] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010135118: str x0, [x8, #2800] : update_irq_load_avg(): : ___update_load_avg(&rq->avg_irq, 1, 1); : trace_pelt_irq_tp(rq); : } : : return ret; : } 0.00 : ffff80001013511c: mov w0, w9 1.14 : ffff800010135120: ldp x29, x30, [sp], #16 0.00 : ffff800010135124: ret : ___update_load_sum(): : sa->last_update_time += delta << 10; 9.90 : ffff800010135128: and x2, x2, #0xfffffffffffffc00 0.00 : ffff80001013512c: add x3, x3, x2 1.58 : ffff800010135130: str x3, [x8, #2752] : accumulate_sum(): : delta += sa->period_contrib; 0.59 : ffff800010135134: ldr w1, [x7, #28] 0.00 : ffff800010135138: add x0, x1, x0 : periods = delta / 1024; /* A period is 1024us (~1ms) */ 0.00 : ffff80001013513c: lsr x6, x0, #10 : if (periods) { 0.00 : ffff800010135140: cbz x6, ffff800010135198 : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.77 : ffff800010135144: cmp x6, #0x7e0 : accumulate_sum(): : sa->load_sum = decay_load(sa->load_sum, periods); 0.40 : ffff800010135148: ldr x2, [x7, #8] : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.00 : ffff80001013514c: b.hi ffff800010135220 // b.pmore : if (unlikely(local_n >= LOAD_AVG_PERIOD)) { 1.67 : ffff800010135150: cmp x6, #0x1f 0.00 : ffff800010135154: ldr w10, [x7, #24] 0.00 : ffff800010135158: ldr x4, [x7, #16] 0.00 : ffff80001013515c: b.hi ffff80001013522c // b.pmore : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010135160: adrp x9, ffff800010d06000 0.00 : ffff800010135164: add x9, x9, #0x450 : mul_u64_u32_shr(): : return (u64)(((unsigned __int128)a * mul) >> shift); 0.22 : ffff800010135168: ldr w1, [x9, w6, uxtw #2] 0.00 : ffff80001013516c: mul x11, x1, x2 0.00 : ffff800010135170: mul x9, x1, x4 0.39 : ffff800010135174: umulh x2, x1, x2 2.28 : ffff800010135178: umulh x4, x1, x4 0.00 : ffff80001013517c: extr x2, x2, x11, #32 0.00 : ffff800010135180: extr x4, x4, x9, #32 0.00 : ffff800010135184: stp x2, x4, [x7, #8] 0.00 : ffff800010135188: mul x1, x10, x1 0.00 : ffff80001013518c: lsr x1, x1, #32 : accumulate_sum(): : delta %= 1024; 0.00 : ffff800010135190: and x0, x0, #0x3ff : sa->util_sum = decay_load((u64)(sa->util_sum), periods); 0.00 : ffff800010135194: str w1, [x7, #24] : sa->period_contrib = delta; 0.41 : ffff800010135198: str w0, [x7, #28] : ___update_load_sum(): : if (!accumulate_sum(delta, sa, load, runnable, running)) 0.00 : ffff80001013519c: cmp w6, #0x0 0.57 : ffff8000101351a0: cset w9, ne // ne = any : if ((s64)delta < 0) { 0.00 : ffff8000101351a4: subs x1, x5, x3 2.69 : ffff8000101351a8: b.mi ffff800010135208 // b.first : delta >>= 10; 0.00 : ffff8000101351ac: lsr x0, x1, #10 : if (!delta) 0.00 : ffff8000101351b0: cbz x0, ffff8000101350f4 0.19 : ffff8000101351b4: b ffff800010135058 : sa->last_update_time = now; 0.00 : ffff8000101351b8: mov x3, x0 0.00 : ffff8000101351bc: str x0, [x8, #2752] 0.00 : ffff8000101351c0: b ffff80001013504c 8.44 : ffff8000101351c4: ldr w12, [x7, #24] : accumulate_sum(): : sa->load_sum += load * contrib; 0.00 : ffff8000101351c8: mov w1, w0 8.04 : ffff8000101351cc: ldr x11, [x7, #16] : ___update_load_sum(): : if (!accumulate_sum(delta, sa, load, runnable, running)) 0.00 : ffff8000101351d0: cmp w14, #0x0 : accumulate_sum(): : sa->load_sum += load * contrib; 0.00 : ffff8000101351d4: add x10, x1, x10 0.00 : ffff8000101351d8: mov w13, w2 : sa->runnable_load_sum += runnable * contrib; 0.00 : ffff8000101351dc: add x1, x1, x11 : sa->util_sum += contrib << SCHED_CAPACITY_SHIFT; 0.00 : ffff8000101351e0: add w0, w12, w0, lsl #10 0.00 : ffff8000101351e4: cinc w9, w9, ne // ne = any : sa->runnable_load_sum += runnable * contrib; 0.00 : ffff8000101351e8: stp x10, x1, [x7, #8] : sa->period_contrib = delta; 1.36 : ffff8000101351ec: stp w0, w13, [x7, #24] 0.00 : ffff8000101351f0: b ffff8000101350f4 : ___update_load_sum(): : sa->last_update_time = now; 0.00 : ffff8000101351f4: str x5, [x8, #2752] : return 0; 0.00 : ffff8000101351f8: mov w9, #0x0 // #0 : update_irq_load_avg(): : } 0.00 : ffff8000101351fc: mov w0, w9 0.00 : ffff800010135200: ldp x29, x30, [sp], #16 0.00 : ffff800010135204: ret : ___update_load_sum(): : sa->last_update_time = now; 0.00 : ffff800010135208: str x5, [x8, #2752] 0.00 : ffff80001013520c: b ffff8000101350f4 : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.00 : ffff800010135210: mov x11, #0x0 // #0 0.00 : ffff800010135214: mov x10, #0x0 // #0 0.00 : ffff800010135218: mov w12, #0x0 // #0 0.00 : ffff80001013521c: b ffff8000101350bc : accumulate_sum(): : sa->runnable_load_sum = 0.00 : ffff800010135220: mov w1, #0x0 // #0 0.00 : ffff800010135224: stp xzr, xzr, [x7, #8] 0.00 : ffff800010135228: b ffff800010135190 : decay_load(): : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff80001013522c: and x11, x6, #0x1f 0.00 : ffff800010135230: adrp x1, ffff800010d06000 0.00 : ffff800010135234: add x1, x1, #0x450 : val >>= local_n / LOAD_AVG_PERIOD; 0.00 : ffff800010135238: lsr w9, w6, #5 0.00 : ffff80001013523c: lsr x2, x2, x9 : mul_u64_u32_shr(): 0.00 : ffff800010135240: ldr w1, [x1, x11, lsl #2] : decay_load(): 0.00 : ffff800010135244: lsr x4, x4, x9 0.00 : ffff800010135248: lsr x10, x10, x9 : mul_u64_u32_shr(): 0.00 : ffff80001013524c: mul x11, x2, x1 0.20 : ffff800010135250: umulh x2, x2, x1 0.00 : ffff800010135254: mul x9, x4, x1 0.00 : ffff800010135258: umulh x4, x4, x1 0.00 : ffff80001013525c: extr x2, x2, x11, #32 0.00 : ffff800010135260: str x2, [x7, #8] 0.00 : ffff800010135264: extr x2, x4, x9, #32 0.00 : ffff800010135268: str x2, [x7, #16] 0.00 : ffff80001013526c: b ffff800010135188 : decay_load(): : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010135270: adrp x5, ffff800010d06000 0.00 : ffff800010135274: and x6, x14, #0x1f 0.00 : ffff800010135278: add x5, x5, #0x450 : val >>= local_n / LOAD_AVG_PERIOD; 0.00 : ffff80001013527c: lsr w1, w14, #5 0.00 : ffff800010135280: lsr x10, x10, x1 0.00 : ffff800010135284: lsr x3, x3, x1 : mul_u64_u32_shr(): 0.00 : ffff800010135288: ldr w12, [x5, x6, lsl #2] : decay_load(): 0.00 : ffff80001013528c: lsr x0, x0, x1 0.00 : ffff800010135290: b ffff80001013509c Percent | Source code & Disassembly of vmlinux for cycles (471 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010139e90 : : queued_spin_lock_slowpath(): : * : v | : : * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' : : * queue : ^--' : : */ : void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) : { 4.03 : ffff800010139e90: stp x29, x30, [sp, #-32]! 0.00 : ffff800010139e94: adrp x7, ffff800011909000 0.00 : ffff800010139e98: add x2, x7, #0x908 : * Wait for in-progress pending->locked hand-overs with a bounded : * number of spins so that we guarantee forward progress. : * : * 0,1,0 -> 0,0,1 : */ : if (val == _Q_PENDING_VAL) { 0.00 : ffff800010139e9c: cmp w1, #0x100 : { 2.96 : ffff800010139ea0: mov x29, sp 0.00 : ffff800010139ea4: ldr x3, [x2] 0.64 : ffff800010139ea8: str x3, [x29, #24] 0.22 : ffff800010139eac: mov x3, #0x0 // #0 0.00 : ffff800010139eb0: mov x3, x0 : if (val == _Q_PENDING_VAL) { 0.00 : ffff800010139eb4: b.eq ffff800010139f78 // b.none : } : : /* : * If we observe any contention; queue. : */ : if (val & ~_Q_LOCKED_MASK) 0.85 : ffff800010139eb8: tst w1, #0xffffff00 0.00 : ffff800010139ebc: b.eq ffff800010139fac // b.none : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.21 : ffff800010139ec0: mov x1, sp : queued_spin_lock_slowpath(): : * queuing. : */ : queue: : lockevent_inc(lock_slowpath); : pv_queue: : node = this_cpu_ptr(&qnodes[0].mcs); 0.00 : ffff800010139ec4: adrp x6, ffff800011529000 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010139ec8: mrs x0, tpidr_el1 : queued_spin_lock_slowpath(): 0.00 : ffff800010139ecc: add x4, x6, #0xa80 0.00 : ffff800010139ed0: add x4, x4, x0 : idx = node->count++; : tail = encode_tail(smp_processor_id(), idx); 0.00 : ffff800010139ed4: adrp x0, ffff80001151d000 0.00 : ffff800010139ed8: add x0, x0, #0x18 : idx = node->count++; 0.00 : ffff800010139edc: ldr w5, [x4, #12] 0.00 : ffff800010139ee0: add w2, w5, #0x1 0.00 : ffff800010139ee4: str w2, [x4, #12] : * 4 nodes will still be extremely unlikely. When that happens, : * we fall back to spinning on the lock directly without using : * any MCS node. This is not the most elegant solution, but is : * simple enough. : */ : if (unlikely(idx >= MAX_NODES)) { 0.00 : ffff800010139ee8: cmp w5, #0x3 : __my_cpu_offset(): 0.00 : ffff800010139eec: mrs x1, tpidr_el1 : queued_spin_lock_slowpath(): : tail = encode_tail(smp_processor_id(), idx); 0.00 : ffff800010139ef0: ldr w10, [x0, x1] : if (unlikely(idx >= MAX_NODES)) { 0.00 : ffff800010139ef4: b.gt ffff80001013a10c : grab_mcs_node(): : return &((struct qnode *)base + idx)->mcs; 0.00 : ffff800010139ef8: sbfiz x9, x5, #4, #32 0.00 : ffff800010139efc: add x8, x4, x9 : queued_spin_lock_slowpath(): : * the actual node. If the compiler is kind enough to reorder these : * stores, then an IRQ could overwrite our assignments. : */ : barrier(); : : node->locked = 0; 0.00 : ffff800010139f00: str wzr, [x8, #8] : node->next = NULL; 0.00 : ffff800010139f04: str xzr, [x4, x9] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010139f08: ldr w0, [x3] : queued_spin_trylock(): : */ : static __always_inline int queued_spin_trylock(struct qspinlock *lock) : { : u32 val = atomic_read(&lock->val); : : if (unlikely(val)) 0.00 : ffff800010139f0c: cbnz w0, ffff80001013a018 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010139f10: b ffff800010139f68 0.00 : ffff800010139f14: b ffff800010139f68 : __lse__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, ) : __CMPXCHG_CASE(w, , , 32, ) : __CMPXCHG_CASE(x, , , 64, ) : __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") : __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 0.00 : ffff800010139f18: mov w1, #0x0 // #0 0.00 : ffff800010139f1c: mov x0, x3 0.00 : ffff800010139f20: mov w2, #0x1 // #1 0.00 : ffff800010139f24: mov w11, w1 0.00 : ffff800010139f28: casa w11, w2, [x3] 0.00 : ffff800010139f2c: mov w0, w11 : queued_spin_lock_slowpath(): : /* : * We touched a (possibly) cold cacheline in the per-cpu queue node; : * attempt the trylock once more in the hope someone let go while we : * weren't watching. : */ : if (queued_spin_trylock(lock)) 0.00 : ffff800010139f30: cbnz w0, ffff80001013a018 : : release: : /* : * release the node : */ : __this_cpu_dec(qnodes[0].mcs.count); 0.21 : ffff800010139f34: add x0, x6, #0xa80 : __my_cpu_offset(): 0.00 : ffff800010139f38: mrs x2, tpidr_el1 : queued_spin_lock_slowpath(): 0.00 : ffff800010139f3c: add x0, x0, #0xc 0.00 : ffff800010139f40: ldr w1, [x0, x2] 0.00 : ffff800010139f44: sub w1, w1, #0x1 0.00 : ffff800010139f48: str w1, [x0, x2] : } 0.00 : ffff800010139f4c: add x0, x7, #0x908 0.00 : ffff800010139f50: ldr x1, [x29, #24] 0.44 : ffff800010139f54: ldr x0, [x0] 0.00 : ffff800010139f58: eor x0, x1, x0 0.00 : ffff800010139f5c: cbnz x0, ffff80001013a1d0 0.00 : ffff800010139f60: ldp x29, x30, [sp], #32 0.00 : ffff800010139f64: ret : __ll_sc__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, , , , , K) : __CMPXCHG_CASE(w, , , 32, , , , , K) : __CMPXCHG_CASE( , , , 64, , , , , L) : __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) : __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 0.00 : ffff800010139f68: mov x1, #0x0 // #0 0.00 : ffff800010139f6c: mov w2, #0x1 // #1 0.00 : ffff800010139f70: b ffff80001013a1d4 0.00 : ffff800010139f74: b ffff800010139f30 : __read_once_size(): 0.00 : ffff800010139f78: ldr w1, [x0] : queued_spin_lock_slowpath(): : val = atomic_cond_read_relaxed(&lock->val, 0.00 : ffff800010139f7c: cmp w1, #0x100 0.00 : ffff800010139f80: b.ne ffff800010139eb8 // b.any : __cmpwait_case_32(): : : [val] "r" (val)); \ : } : : __CMPWAIT_CASE(w, b, 8); : __CMPWAIT_CASE(w, h, 16); : __CMPWAIT_CASE(w, , 32); 0.00 : ffff800010139f84: mov x0, #0x100 // #256 0.00 : ffff800010139f88: sevl 0.00 : ffff800010139f8c: wfe 0.00 : ffff800010139f90: ldxr w1, [x3] 0.00 : ffff800010139f94: eor w1, w1, w0 0.00 : ffff800010139f98: cbnz w1, ffff800010139fa0 0.00 : ffff800010139f9c: wfe : __read_once_size(): 0.00 : ffff800010139fa0: ldr w1, [x3] : queued_spin_lock_slowpath(): : if (val & ~_Q_LOCKED_MASK) 0.00 : ffff800010139fa4: tst w1, #0xffffff00 0.00 : ffff800010139fa8: b.ne ffff800010139ec0 // b.any : arch_static_branch_jump(): 0.22 : ffff800010139fac: b ffff800010139fc0 0.00 : ffff800010139fb0: b ffff800010139fc0 : __lse_atomic_fetch_or_acquire(): : ATOMIC_FETCH_OPS(or, ldset) 0.21 : ffff800010139fb4: mov w0, #0x100 // #256 0.22 : ffff800010139fb8: ldseta w0, w0, [x3] 67.93 : ffff800010139fbc: b ffff800010139fc8 : __ll_sc_atomic_fetch_or_acquire(): : ATOMIC_OPS(or, orr, K) 0.00 : ffff800010139fc0: mov w1, #0x100 // #256 0.00 : ffff800010139fc4: b ffff80001013a1f0 : queued_spin_lock_slowpath(): : if (unlikely(val & ~_Q_LOCKED_MASK)) { 0.00 : ffff800010139fc8: tst w0, #0xffffff00 0.00 : ffff800010139fcc: b.ne ffff80001013a154 // b.any : if (val & _Q_LOCKED_MASK) 0.00 : ffff800010139fd0: cbnz w0, ffff800010139ffc : __write_once_size(): : : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; 0.44 : ffff800010139fd4: mov w0, #0x1 // #1 0.00 : ffff800010139fd8: strh w0, [x3] 0.00 : ffff800010139fdc: b ffff800010139f4c : queued_spin_lock_slowpath(): : atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_MASK)); 0.00 : ffff800010139fe0: sxtw x0, w0 : __cmpwait_case_32(): 0.00 : ffff800010139fe4: sevl 0.00 : ffff800010139fe8: wfe 2.78 : ffff800010139fec: ldxr w1, [x3] 0.00 : ffff800010139ff0: eor w1, w1, w0 0.00 : ffff800010139ff4: cbnz w1, ffff800010139ffc 1.30 : ffff800010139ff8: wfe : queued_spin_lock_slowpath(): 8.34 : ffff800010139ffc: ldar w0, [x3] 7.06 : ffff80001013a000: str w0, [x29, #16] 0.00 : ffff80001013a004: and w1, w0, #0xff 0.00 : ffff80001013a008: cbnz w1, ffff800010139fe0 : __write_once_size(): 0.00 : ffff80001013a00c: mov w0, #0x1 // #1 0.00 : ffff80001013a010: strh w0, [x3] 0.00 : ffff80001013a014: b ffff800010139f4c : encode_tail(): : tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET; 0.43 : ffff80001013a018: add w10, w10, #0x1 : tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */ 0.00 : ffff80001013a01c: lsl w5, w5, #16 0.00 : ffff80001013a020: orr w10, w5, w10, lsl #18 : queued_spin_lock_slowpath(): : smp_wmb(); 0.00 : ffff80001013a024: dmb ishst : xchg_tail(): : return (u32)xchg_relaxed(&lock->tail, 0.00 : ffff80001013a028: lsr w1, w10, #16 : __xchg_case_16(): : __XCHG_CASE(w, h, , 16, , , , , , ) 0.00 : ffff80001013a02c: add x5, x3, #0x2 0.00 : ffff80001013a030: prfm pstl1strm, [x5] 0.65 : ffff80001013a034: ldxrh w0, [x5] 0.00 : ffff80001013a038: stxrh w2, w1, [x5] 0.00 : ffff80001013a03c: cbnz w2, ffff80001013a034 0.00 : ffff80001013a040: and w0, w0, #0xffff : xchg_tail(): : tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET; 0.00 : ffff80001013a044: lsl w1, w0, #16 : queued_spin_lock_slowpath(): : if (old & _Q_TAIL_MASK) { 0.00 : ffff80001013a048: cbz w1, ffff80001013a0b0 : decode_tail(): : int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1; 0.00 : ffff80001013a04c: lsr w1, w1, #18 : return per_cpu_ptr(&qnodes[idx].mcs, cpu); 0.00 : ffff80001013a050: adrp x5, ffff800011909000 : int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1; 0.00 : ffff80001013a054: sub w2, w1, #0x1 : return per_cpu_ptr(&qnodes[idx].mcs, cpu); 0.00 : ffff80001013a058: add x5, x5, #0x928 0.00 : ffff80001013a05c: add x1, x6, #0xa80 0.00 : ffff80001013a060: ubfiz x0, x0, #4, #2 0.00 : ffff80001013a064: add x1, x1, x0 : queued_spin_lock_slowpath(): : arch_mcs_spin_lock_contended(&node->locked); 0.00 : ffff80001013a068: add x0, x8, #0x8 : decode_tail(): : return per_cpu_ptr(&qnodes[idx].mcs, cpu); 0.00 : ffff80001013a06c: ldr x2, [x5, w2, sxtw #3] : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001013a070: str x8, [x1, x2] : queued_spin_lock_slowpath(): : arch_mcs_spin_lock_contended(&node->locked); 0.00 : ffff80001013a074: ldar w1, [x0] 0.00 : ffff80001013a078: cbnz w1, ffff80001013a0a0 : __cmpwait_case_32(): : __CMPWAIT_CASE(w, , 32); 0.00 : ffff80001013a07c: mov x1, #0x0 // #0 0.00 : ffff80001013a080: sevl 0.00 : ffff80001013a084: wfe 0.00 : ffff80001013a088: ldxr w2, [x0] 0.00 : ffff80001013a08c: eor w2, w2, w1 0.00 : ffff80001013a090: cbnz w2, ffff80001013a098 0.00 : ffff80001013a094: wfe : queued_spin_lock_slowpath(): 0.00 : ffff80001013a098: ldar w2, [x0] 0.00 : ffff80001013a09c: cbz w2, ffff80001013a080 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001013a0a0: ldr x11, [x4, x9] : queued_spin_lock_slowpath(): : if (next) 0.00 : ffff80001013a0a4: cbz x11, ffff80001013a0b0 : prefetchw(): : } : : #define ARCH_HAS_PREFETCHW : static inline void prefetchw(const void *ptr) : { : asm volatile("prfm pstl1keep, %a0\n" : : "p" (ptr)); 0.00 : ffff80001013a0a8: prfm pstl1keep, [x11] 0.00 : ffff80001013a0ac: b ffff80001013a0d4 : queued_spin_lock_slowpath(): : next = NULL; 0.00 : ffff80001013a0b0: mov x11, #0x0 // #0 0.00 : ffff80001013a0b4: b ffff80001013a0d4 : val = atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK)); 0.00 : ffff80001013a0b8: sxtw x5, w5 : __cmpwait_case_32(): 0.00 : ffff80001013a0bc: sevl 0.00 : ffff80001013a0c0: wfe 0.00 : ffff80001013a0c4: ldxr w0, [x3] 0.00 : ffff80001013a0c8: eor w0, w0, w5 0.00 : ffff80001013a0cc: cbnz w0, ffff80001013a0d4 0.00 : ffff80001013a0d0: wfe : queued_spin_lock_slowpath(): 0.63 : ffff80001013a0d4: ldar w5, [x3] 0.21 : ffff80001013a0d8: str w5, [x29, #20] 0.00 : ffff80001013a0dc: and w0, w5, #0xffff 0.00 : ffff80001013a0e0: cbnz w0, ffff80001013a0b8 : if ((val & _Q_TAIL_MASK) == tail) { 0.00 : ffff80001013a0e4: and w1, w5, #0xffff0000 0.00 : ffff80001013a0e8: cmp w1, w10 0.00 : ffff80001013a0ec: b.eq ffff80001013a194 // b.none : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff80001013a0f0: mov w0, #0x1 // #1 0.00 : ffff80001013a0f4: strb w0, [x3] : queued_spin_lock_slowpath(): : if (!next) 0.00 : ffff80001013a0f8: cbz x11, ffff80001013a164 : arch_mcs_spin_unlock_contended(&next->locked); 0.00 : ffff80001013a0fc: mov w0, #0x1 // #1 0.00 : ffff80001013a100: add x1, x11, #0x8 0.00 : ffff80001013a104: stlr w0, [x1] 0.00 : ffff80001013a108: b ffff800010139f34 : __ll_sc__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 0.00 : ffff80001013a10c: mov x5, #0x0 // #0 0.00 : ffff80001013a110: mov w4, #0x1 // #1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001013a114: ldr w0, [x3] : queued_spin_trylock(): 0.00 : ffff80001013a118: cbnz w0, ffff80001013a140 : arch_static_branch_jump(): 0.00 : ffff80001013a11c: b ffff80001013a148 0.00 : ffff80001013a120: b ffff80001013a148 : __lse__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 0.00 : ffff80001013a124: mov w1, #0x0 // #0 0.00 : ffff80001013a128: mov x0, x3 0.00 : ffff80001013a12c: mov w2, #0x1 // #1 0.00 : ffff80001013a130: mov w8, w1 0.00 : ffff80001013a134: casa w8, w2, [x3] 0.00 : ffff80001013a138: mov w0, w8 : queued_spin_lock_slowpath(): : while (!queued_spin_trylock(lock)) 0.00 : ffff80001013a13c: cbz w0, ffff800010139f34 : cpu_relax(): : asm volatile("yield" ::: "memory"); 0.00 : ffff80001013a140: yield 0.00 : ffff80001013a144: b ffff80001013a114 : __ll_sc__cmpxchg_case_acq_32(): 0.00 : ffff80001013a148: b ffff80001013a208 : queued_spin_lock_slowpath(): 0.00 : ffff80001013a14c: cbz w0, ffff800010139f34 0.00 : ffff80001013a150: b ffff80001013a140 : if (!(val & _Q_PENDING_MASK)) 0.00 : ffff80001013a154: tst w0, #0xff00 0.00 : ffff80001013a158: b.ne ffff800010139ec0 // b.any : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff80001013a15c: strb wzr, [x3, #1] 0.00 : ffff80001013a160: b ffff800010139ec0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001013a164: ldr x11, [x4, x9] : __cmpwait_case_64(): : __CMPWAIT_CASE( , , 64); 0.00 : ffff80001013a168: mov x0, #0x0 // #0 : queued_spin_lock_slowpath(): : next = smp_cond_load_relaxed(&node->next, (VAL)); 0.00 : ffff80001013a16c: cbnz x11, ffff80001013a0fc : __cmpwait_case_64(): 0.00 : ffff80001013a170: sevl 0.00 : ffff80001013a174: wfe 0.00 : ffff80001013a178: ldxr x1, [x8] 0.00 : ffff80001013a17c: eor x1, x1, x0 0.00 : ffff80001013a180: cbnz x1, ffff80001013a188 0.00 : ffff80001013a184: wfe : __read_once_size(): 0.00 : ffff80001013a188: ldr x11, [x8] : queued_spin_lock_slowpath(): 0.00 : ffff80001013a18c: cbz x11, ffff80001013a170 0.00 : ffff80001013a190: b ffff80001013a0fc : atomic_cmpxchg_relaxed(): : #if defined(arch_atomic_cmpxchg_relaxed) : static inline int : atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) : { : kasan_check_write(v, sizeof(*v)); : return arch_atomic_cmpxchg_relaxed(v, old, new); 0.00 : ffff80001013a194: sxtw x1, w5 : arch_static_branch_jump(): 0.00 : ffff80001013a198: b ffff80001013a1c0 0.00 : ffff80001013a19c: b ffff80001013a1c0 : __lse__cmpxchg_case_32(): : __CMPXCHG_CASE(w, , , 32, ) 0.00 : ffff80001013a1a0: mov w2, #0x1 // #1 0.00 : ffff80001013a1a4: mov x0, x3 0.00 : ffff80001013a1a8: mov w10, w1 0.00 : ffff80001013a1ac: cas w10, w2, [x3] 0.00 : ffff80001013a1b0: mov w0, w10 : queued_spin_lock_slowpath(): : if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL)) 0.00 : ffff80001013a1b4: cmp w0, w5 0.00 : ffff80001013a1b8: b.eq ffff800010139f34 // b.none 0.00 : ffff80001013a1bc: b ffff80001013a0f0 : __ll_sc__cmpxchg_case_32(): : __CMPXCHG_CASE(w, , , 32, , , , , K) 0.00 : ffff80001013a1c0: and x1, x1, #0xffffffff 0.00 : ffff80001013a1c4: mov w2, #0x1 // #1 0.00 : ffff80001013a1c8: b ffff80001013a224 0.00 : ffff80001013a1cc: b ffff80001013a1b4 : queued_spin_lock_slowpath(): : } 0.00 : ffff80001013a1d0: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (506 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010178338 : : clockevents_program_event(): : * : * Returns 0 on success, -ETIME when the event is in the past. : */ : int clockevents_program_event(struct clock_event_device *dev, ktime_t expires, : bool force) : { 0.18 : ffff800010178338: stp x29, x30, [sp, #-48]! 0.00 : ffff80001017833c: mov x29, sp : unsigned long long clc; : int64_t delta; : int rc; : : if (WARN_ON_ONCE(expires < 0)) 0.00 : ffff800010178340: tbnz x1, #63, ffff8000101783a8 0.19 : ffff800010178344: str x19, [x29, #16] 0.00 : ffff800010178348: mov x19, x0 : return -ETIME; : : dev->next_event = expires; : : if (clockevent_state_shutdown(dev)) : return 0; 0.00 : ffff80001017834c: mov w0, #0x0 // #0 : if (clockevent_state_shutdown(dev)) 1.60 : ffff800010178350: ldr w4, [x19, #56] : dev->next_event = expires; 8.00 : ffff800010178354: str x1, [x19, #24] : if (clockevent_state_shutdown(dev)) 0.00 : ffff800010178358: cmp w4, #0x1 0.00 : ffff80001017835c: b.eq ffff80001017839c // b.none 9.20 : ffff800010178360: stp x20, x21, [x29, #24] : : /* We must be in ONESHOT state here */ : WARN_ONCE(!clockevent_state_oneshot(dev), "Current state: %d\n", 0.00 : ffff800010178364: cmp w4, #0x3 0.00 : ffff800010178368: and w21, w2, #0xff 0.00 : ffff80001017836c: mov x20, x1 2.54 : ffff800010178370: b.ne ffff8000101783b8 // b.any : clockevent_get_state(dev)); : : /* Shortcut for clockevent devices that can deal with ktime. */ : if (dev->features & CLOCK_EVT_FEAT_KTIME) 0.00 : ffff800010178374: ldr w0, [x19, #60] 0.00 : ffff800010178378: tbz w0, #2, ffff8000101783e8 : return dev->set_next_ktime(expires, dev); 0.00 : ffff80001017837c: ldr x2, [x19, #16] 0.00 : ffff800010178380: mov x1, x19 0.00 : ffff800010178384: mov x0, x20 0.00 : ffff800010178388: blr x2 0.00 : ffff80001017838c: ldr x21, [x29, #32] 0.00 : ffff800010178390: ldp x19, x20, [x29, #16] : : clc = ((unsigned long long) delta * dev->mult) >> dev->shift; : rc = dev->set_next_event((unsigned long) clc, dev); : : return (rc && force) ? clockevents_program_min_delta(dev) : rc; : } 0.00 : ffff800010178394: ldp x29, x30, [sp], #48 0.00 : ffff800010178398: ret 0.00 : ffff80001017839c: ldr x19, [x29, #16] 0.00 : ffff8000101783a0: ldp x29, x30, [sp], #48 0.00 : ffff8000101783a4: ret : if (WARN_ON_ONCE(expires < 0)) 0.00 : ffff8000101783a8: brk #0x800 : return -ETIME; 0.00 : ffff8000101783ac: mov w0, #0xffffffc2 // #-62 : } 0.00 : ffff8000101783b0: ldp x29, x30, [sp], #48 0.00 : ffff8000101783b4: ret : WARN_ONCE(!clockevent_state_oneshot(dev), "Current state: %d\n", 0.00 : ffff8000101783b8: adrp x1, ffff800011ad8000 0.00 : ffff8000101783bc: add x1, x1, #0x4d1 0.00 : ffff8000101783c0: ldrb w0, [x1, #1] 0.00 : ffff8000101783c4: cbnz w0, ffff800010178374 0.00 : ffff8000101783c8: mov w2, #0x1 // #1 0.00 : ffff8000101783cc: adrp x0, ffff8000111c9000 0.00 : ffff8000101783d0: strb w2, [x1, #1] 0.00 : ffff8000101783d4: add x0, x0, #0x5f0 0.00 : ffff8000101783d8: mov w1, w4 0.00 : ffff8000101783dc: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff8000101783e0: brk #0x800 0.00 : ffff8000101783e4: b ffff800010178374 : delta = ktime_to_ns(ktime_sub(expires, ktime_get())); 5.73 : ffff8000101783e8: bl ffff80001016c938 0.00 : ffff8000101783ec: sub x1, x20, x0 : if (delta <= 0) 0.00 : ffff8000101783f0: cmp x1, #0x0 0.00 : ffff8000101783f4: b.le ffff80001017843c : delta = max(delta, (int64_t) dev->min_delta_ns); 0.00 : ffff8000101783f8: ldp x20, x2, [x19, #32] : clc = ((unsigned long long) delta * dev->mult) >> dev->shift; 1.97 : ffff8000101783fc: ldp w4, w0, [x19, #48] : delta = min(delta, (int64_t) dev->max_delta_ns); 0.00 : ffff800010178400: cmp x20, x1 0.00 : ffff800010178404: csel x20, x20, x1, le : rc = dev->set_next_event((unsigned long) clc, dev); 3.81 : ffff800010178408: ldr x3, [x19, #8] : delta = max(delta, (int64_t) dev->min_delta_ns); 0.00 : ffff80001017840c: cmp x20, x2 : rc = dev->set_next_event((unsigned long) clc, dev); 0.00 : ffff800010178410: mov x1, x19 : delta = max(delta, (int64_t) dev->min_delta_ns); 0.00 : ffff800010178414: csel x20, x20, x2, ge // ge = tcont : clc = ((unsigned long long) delta * dev->mult) >> dev->shift; 0.00 : ffff800010178418: mul x20, x20, x4 : rc = dev->set_next_event((unsigned long) clc, dev); 0.00 : ffff80001017841c: lsr x0, x20, x0 0.00 : ffff800010178420: blr x3 : return (rc && force) ? clockevents_program_min_delta(dev) : rc; 5.36 : ffff800010178424: cmp w0, #0x0 0.00 : ffff800010178428: ccmp w21, #0x0, #0x4, ne // ne = any 0.00 : ffff80001017842c: b.ne ffff800010178440 // b.any 28.14 : ffff800010178430: ldp x19, x20, [x29, #16] 33.27 : ffff800010178434: ldr x21, [x29, #32] 0.00 : ffff800010178438: b ffff800010178394 : return force ? clockevents_program_min_delta(dev) : -ETIME; 0.00 : ffff80001017843c: cbz w21, ffff800010178454 : return (rc && force) ? clockevents_program_min_delta(dev) : rc; 0.00 : ffff800010178440: mov x0, x19 0.00 : ffff800010178444: bl ffff8000101779a0 0.00 : ffff800010178448: ldp x19, x20, [x29, #16] 0.00 : ffff80001017844c: ldr x21, [x29, #32] 0.00 : ffff800010178450: b ffff800010178394 : return force ? clockevents_program_min_delta(dev) : -ETIME; 0.00 : ffff800010178454: mov w0, #0xffffffc2 // #-62 0.00 : ffff800010178458: ldr x21, [x29, #32] 0.00 : ffff80001017845c: ldp x19, x20, [x29, #16] 0.00 : ffff800010178460: b ffff800010178394 Percent | Source code & Disassembly of vmlinux for cycles (467 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012f9c0 <__wake_up>: : __wake_up(): : * If this function wakes up a task, it executes a full memory barrier before : * accessing the task state. : */ : void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, : int nr_exclusive, void *key) : { 40.52 : ffff80001012f9c0: stp x29, x30, [sp, #-16]! : __wake_up_common_lock(wq_head, mode, nr_exclusive, 0, key); 0.00 : ffff80001012f9c4: mov x4, x3 0.00 : ffff80001012f9c8: mov w3, #0x0 // #0 : { 0.59 : ffff80001012f9cc: mov x29, sp : __wake_up_common_lock(wq_head, mode, nr_exclusive, 0, key); 56.31 : ffff80001012f9d0: bl ffff80001012f8f8 <__wake_up_common_lock> : } 2.58 : ffff80001012f9d4: ldp x29, x30, [sp], #16 0.00 : ffff80001012f9d8: ret Percent | Source code & Disassembly of vmlinux for cycles (474 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101691d8 <__next_timer_interrupt>: : __next_timer_interrupt(): : /* : * Search the first expiring timer in the various clock levels. Caller must : * hold base->lock. : */ : static unsigned long __next_timer_interrupt(struct timer_base *base) : { 0.00 : ffff8000101691d8: stp x29, x30, [sp, #-96]! 0.00 : ffff8000101691dc: mov x29, sp 0.00 : ffff8000101691e0: stp x19, x20, [sp, #16] 0.22 : ffff8000101691e4: stp x21, x22, [sp, #32] : unsigned long clk, next, adj; : unsigned lvl, offset = 0; : : next = base->clk + NEXT_TIMER_MAX_DELTA; 0.00 : ffff8000101691e8: mov x22, #0x3fffffff // #1073741823 : { 0.84 : ffff8000101691ec: stp x23, x24, [sp, #48] : next = base->clk + NEXT_TIMER_MAX_DELTA; 0.00 : ffff8000101691f0: mov x21, #0x0 // #0 : { 0.00 : ffff8000101691f4: stp x25, x26, [sp, #64] 0.00 : ffff8000101691f8: add x23, x0, #0x28 0.00 : ffff8000101691fc: str x27, [sp, #80] : next = base->clk + NEXT_TIMER_MAX_DELTA; 0.00 : ffff800010169200: mov w26, #0x0 // #0 : next_pending_bucket(): : return pos < start ? pos + LVL_SIZE - start : -1; 0.00 : ffff800010169204: mov w24, #0x40 // #64 : __next_timer_interrupt(): : next = base->clk + NEXT_TIMER_MAX_DELTA; 3.94 : ffff800010169208: ldr x20, [x0, #16] 0.00 : ffff80001016920c: add x22, x20, x22 0.00 : ffff800010169210: b ffff800010169248 <__next_timer_interrupt+0x70> : clk = base->clk; : for (lvl = 0; lvl < LVL_DEPTH; lvl++, offset += LVL_SIZE) { : int pos = next_pending_bucket(base, offset, clk & LVL_MASK); : : if (pos >= 0) { : unsigned long tmp = clk + (unsigned long) pos; 7.41 : ffff800010169214: add x0, x20, w1, sxtw : if (pos >= 0) { 0.00 : ffff800010169218: tbnz w1, #31, ffff80001016922c <__next_timer_interrupt+0x54> : : tmp <<= LVL_SHIFT(lvl); 19.75 : ffff80001016921c: lsl x0, x0, x26 : if (time_before(tmp, next)) 1.49 : ffff800010169220: sub x1, x0, x22 : next = tmp; 0.00 : ffff800010169224: cmp x1, #0x0 0.00 : ffff800010169228: csel x22, x22, x0, ge // ge = tcont : * from LVL2 to LVL3. : * : * So the simple check whether the lower bits of the current : * level are 0 or not is sufficient for all cases. : */ : adj = clk & LVL_CLK_MASK ? 1 : 0; 5.06 : ffff80001016922c: tst x20, #0x7 : clk >>= LVL_CLK_SHIFT; 0.00 : ffff800010169230: lsr x20, x20, #3 : clk += adj; 0.00 : ffff800010169234: cinc x20, x20, ne // ne = any 0.00 : ffff800010169238: add w26, w26, #0x3 0.00 : ffff80001016923c: mov x21, x25 : for (lvl = 0; lvl < LVL_DEPTH; lvl++, offset += LVL_SIZE) { 0.00 : ffff800010169240: cmp x25, #0x240 0.00 : ffff800010169244: b.eq ffff800010169298 <__next_timer_interrupt+0xc0> // b.none : int pos = next_pending_bucket(base, offset, clk & LVL_MASK); 13.16 : ffff800010169248: and w19, w20, #0x3f 0.00 : ffff80001016924c: add x25, x21, #0x40 : next_pending_bucket(): : unsigned pos, start = offset + clk; 0.00 : ffff800010169250: add w27, w19, w21 : pos = find_next_bit(base->pending_map, end, start); 0.00 : ffff800010169254: mov x1, x25 9.36 : ffff800010169258: mov x2, x27 0.00 : ffff80001016925c: mov x0, x23 0.00 : ffff800010169260: bl ffff80001048daa8 : return pos - start; 6.56 : ffff800010169264: sub w1, w0, w27 : if (pos < end) 0.00 : ffff800010169268: add w2, w21, #0x40 0.00 : ffff80001016926c: cmp w2, w0 0.00 : ffff800010169270: b.hi ffff800010169214 <__next_timer_interrupt+0x3c> // b.pmore : pos = find_next_bit(base->pending_map, start, offset); 1.09 : ffff800010169274: mov x1, x27 0.00 : ffff800010169278: mov x2, x21 0.00 : ffff80001016927c: mov x0, x23 0.00 : ffff800010169280: bl ffff80001048daa8 : return pos < start ? pos + LVL_SIZE - start : -1; 18.44 : ffff800010169284: sub w1, w24, w27 0.00 : ffff800010169288: cmp w27, w0 0.00 : ffff80001016928c: add w1, w1, w0 0.00 : ffff800010169290: b.hi ffff800010169214 <__next_timer_interrupt+0x3c> // b.pmore 3.83 : ffff800010169294: b ffff80001016922c <__next_timer_interrupt+0x54> : __next_timer_interrupt(): : } : return next; : } 0.00 : ffff800010169298: mov x0, x22 1.27 : ffff80001016929c: ldr x27, [sp, #80] 4.77 : ffff8000101692a0: ldp x19, x20, [sp, #16] 1.09 : ffff8000101692a4: ldp x21, x22, [sp, #32] 0.22 : ffff8000101692a8: ldp x23, x24, [sp, #48] 1.50 : ffff8000101692ac: ldp x25, x26, [sp, #64] 0.00 : ffff8000101692b0: ldp x29, x30, [sp], #96 0.00 : ffff8000101692b4: ret Percent | Source code & Disassembly of vmlinux for cycles (578 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001048daa8 : : find_next_bit(): : /* : * Find the next set bit in a memory region. : */ : unsigned long find_next_bit(const unsigned long *addr, unsigned long size, : unsigned long offset) : { 8.52 : ffff80001048daa8: stp x29, x30, [sp, #-16]! : return _find_next_bit(addr, NULL, size, offset, 0UL, 0); 9.91 : ffff80001048daac: mov x3, x2 0.53 : ffff80001048dab0: mov x4, #0x0 // #0 0.86 : ffff80001048dab4: mov x2, x1 : { 70.85 : ffff80001048dab8: mov x29, sp : return _find_next_bit(addr, NULL, size, offset, 0UL, 0); 2.95 : ffff80001048dabc: mov x1, #0x0 // #0 0.00 : ffff80001048dac0: bl ffff80001048d988 <_find_next_bit.constprop.1> : } 5.33 : ffff80001048dac4: ldp x29, x30, [sp], #16 1.05 : ffff80001048dac8: ret Percent | Source code & Disassembly of vmlinux for cycles (422 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001071d288 : : arm_smmu_unmap(): : : static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, : size_t size, struct iommu_iotlb_gather *gather) : { : struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); : struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 34.15 : ffff80001071d288: ldur x0, [x0, #-96] : : if (!ops) 0.96 : ffff80001071d28c: cbz x0, ffff80001071d2a8 : { 16.36 : ffff80001071d290: stp x29, x30, [sp, #-16]! 0.00 : ffff80001071d294: mov x29, sp : return 0; : : return ops->unmap(ops, iova, size, gather); 14.74 : ffff80001071d298: ldr x4, [x0, #8] 7.11 : ffff80001071d29c: blr x4 : } 26.44 : ffff80001071d2a0: ldp x29, x30, [sp], #16 0.25 : ffff80001071d2a4: ret : return 0; 0.00 : ffff80001071d2a8: mov x0, #0x0 // #0 : } 0.00 : ffff80001071d2ac: ret Percent | Source code & Disassembly of vmlinux for cycles (419 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104b0ee0 <__sbq_wake_up>: : __sbq_wake_up(): : : return NULL; : } : : static bool __sbq_wake_up(struct sbitmap_queue *sbq) : { 0.00 : ffff8000104b0ee0: mov x7, x0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.16 : ffff8000104b0ee4: ldr w0, [x0, #48] : sbq_wake_ptr(): : if (!atomic_read(&sbq->ws_active)) 3.59 : ffff8000104b0ee8: cbz w0, ffff8000104b0f24 <__sbq_wake_up+0x44> : __read_once_size(): 0.00 : ffff8000104b0eec: ldr w0, [x7, #36] : sbq_wake_ptr(): : struct sbq_wait_state *ws = &sbq->ws[wake_index]; 0.00 : ffff8000104b0ef0: mov w1, #0x8 // #8 0.00 : ffff8000104b0ef4: ldr x8, [x7, #40] 0.00 : ffff8000104b0ef8: sbfiz x3, x0, #6, #32 : sbq_index_inc(): : void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, : unsigned int cpu); : : static inline int sbq_index_inc(int index) : { : return (index + 1) & (SBQ_WAIT_QUEUES - 1); 0.00 : ffff8000104b0efc: add w6, w0, #0x1 : sbq_wake_ptr(): 0.00 : ffff8000104b0f00: add x3, x8, x3 : waitqueue_active(): : * Also note that this 'optimization' trades a spin_lock() for an smp_mb(), : * which (when the lock is uncontended) are of roughly equal cost. : */ : static inline int waitqueue_active(struct wait_queue_head *wq_head) : { : return !list_empty(&wq_head->head); 0.00 : ffff8000104b0f04: add x4, x3, #0x8 0.00 : ffff8000104b0f08: add x5, x4, #0x8 : __read_once_size(): 0.00 : ffff8000104b0f0c: ldr x2, [x3, #16] : sbq_wake_ptr(): : if (waitqueue_active(&ws->wait)) { 0.00 : ffff8000104b0f10: cmp x5, x2 0.00 : ffff8000104b0f14: b.ne ffff8000104b0f2c <__sbq_wake_up+0x4c> // b.any : sbq_index_inc(): 0.00 : ffff8000104b0f18: and w0, w6, #0x7 : sbq_wake_ptr(): : for (i = 0; i < SBQ_WAIT_QUEUES; i++) { 0.00 : ffff8000104b0f1c: subs w1, w1, #0x1 0.00 : ffff8000104b0f20: b.ne ffff8000104b0ef8 <__sbq_wake_up+0x18> // b.any : __sbq_wake_up(): : unsigned int wake_batch; : int wait_cnt; : : ws = sbq_wake_ptr(sbq); : if (!ws) : return false; 94.26 : ffff8000104b0f24: mov w0, #0x0 // #0 : : return true; : } : : return false; : } 0.00 : ffff8000104b0f28: ret : __read_once_size(): 0.00 : ffff8000104b0f2c: ldr w1, [x7, #36] : sbq_wake_ptr(): : if (wake_index != atomic_read(&sbq->wake_index)) 0.00 : ffff8000104b0f30: cmp w1, w0 0.00 : ffff8000104b0f34: b.eq ffff8000104b0f3c <__sbq_wake_up+0x5c> // b.none : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104b0f38: str w0, [x7, #36] : __sbq_wake_up(): : if (!ws) 0.00 : ffff8000104b0f3c: cbz x3, ffff8000104b0f24 <__sbq_wake_up+0x44> : { 0.00 : ffff8000104b0f40: stp x29, x30, [sp, #-16]! 0.00 : ffff8000104b0f44: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000104b0f48: b ffff8000104b0f74 <__sbq_wake_up+0x94> 0.00 : ffff8000104b0f4c: b ffff8000104b0f74 <__sbq_wake_up+0x94> : __lse_atomic_sub_return(): : } : : ATOMIC_OP_SUB_RETURN(_relaxed, ) : ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC_OP_SUB_RETURN(_release, l, "memory") : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000104b0f50: mov w5, #0x1 // #1 0.00 : ffff8000104b0f54: neg w5, w5 0.00 : ffff8000104b0f58: ldaddal w5, w0, [x3] 0.00 : ffff8000104b0f5c: add w5, w5, w0 : __sbq_wake_up(): : if (wait_cnt <= 0) { 0.00 : ffff8000104b0f60: cmp w5, #0x0 0.00 : ffff8000104b0f64: b.le ffff8000104b0f84 <__sbq_wake_up+0xa4> : return false; 0.00 : ffff8000104b0f68: mov w0, #0x0 // #0 : } 0.00 : ffff8000104b0f6c: ldp x29, x30, [sp], #16 0.00 : ffff8000104b0f70: ret : __ll_sc_atomic_sub_return(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000104b0f74: mov w0, #0x1 // #1 0.00 : ffff8000104b0f78: b ffff8000104b14e0 <__sbitmap_queue_get+0x248> : __sbq_wake_up(): : if (wait_cnt <= 0) { 0.00 : ffff8000104b0f7c: cmp w5, #0x0 0.00 : ffff8000104b0f80: b.gt ffff8000104b0f68 <__sbq_wake_up+0x88> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000104b0f84: ldr w6, [x7, #32] : __sbq_wake_up(): : smp_mb__before_atomic(); 0.00 : ffff8000104b0f88: dmb ish : atomic_cmpxchg(): : #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) : static inline int : atomic_cmpxchg(atomic_t *v, int old, int new) : { : kasan_check_write(v, sizeof(*v)); : return arch_atomic_cmpxchg(v, old, new); 0.00 : ffff8000104b0f8c: sxtw x1, w5 : arch_static_branch_jump(): 0.00 : ffff8000104b0f90: b ffff8000104b0fb8 <__sbq_wake_up+0xd8> 0.00 : ffff8000104b0f94: b ffff8000104b0fb8 <__sbq_wake_up+0xd8> : __lse__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 0.00 : ffff8000104b0f98: mov x0, x3 0.00 : ffff8000104b0f9c: mov w1, w5 0.00 : ffff8000104b0fa0: mov w2, w6 0.00 : ffff8000104b0fa4: mov w8, w1 0.00 : ffff8000104b0fa8: casal w8, w2, [x3] 0.00 : ffff8000104b0fac: mov w0, w8 0.00 : ffff8000104b0fb0: mov w1, w0 0.00 : ffff8000104b0fb4: b ffff8000104b0fc4 <__sbq_wake_up+0xe4> : __ll_sc__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 0.00 : ffff8000104b0fb8: and x1, x1, #0xffffffff 0.00 : ffff8000104b0fbc: b ffff8000104b14fc <__sbitmap_queue_get+0x264> 0.00 : ffff8000104b0fc0: mov w1, w0 : __sbq_wake_up(): : return true; 0.00 : ffff8000104b0fc4: mov w0, #0x1 // #1 : if (ret == wait_cnt) { 0.00 : ffff8000104b0fc8: cmp w1, w5 0.00 : ffff8000104b0fcc: b.ne ffff8000104b0f6c <__sbq_wake_up+0x8c> // b.any : __read_once_size(): 0.00 : ffff8000104b0fd0: mov x0, x7 0.00 : ffff8000104b0fd4: ldr w1, [x0, #36]! : sbq_index_inc(): 0.00 : ffff8000104b0fd8: add w2, w1, #0x1 : atomic_cmpxchg(): 0.00 : ffff8000104b0fdc: sxtw x3, w1 : sbq_index_inc(): 0.00 : ffff8000104b0fe0: and w2, w2, #0x7 : arch_static_branch_jump(): 0.00 : ffff8000104b0fe4: b ffff8000104b1014 <__sbq_wake_up+0x134> 0.00 : ffff8000104b0fe8: b ffff8000104b1014 <__sbq_wake_up+0x134> : __lse__cmpxchg_case_mb_32(): 0.00 : ffff8000104b0fec: mov x3, x0 0.00 : ffff8000104b0ff0: mov w5, w1 0.00 : ffff8000104b0ff4: casal w5, w2, [x3] 0.00 : ffff8000104b0ff8: mov w0, w5 : __sbq_wake_up(): : wake_up_nr(&ws->wait, wake_batch); 0.00 : ffff8000104b0ffc: mov x3, #0x0 // #0 0.00 : ffff8000104b1000: mov w2, w6 0.00 : ffff8000104b1004: mov w1, #0x3 // #3 0.00 : ffff8000104b1008: mov x0, x4 0.00 : ffff8000104b100c: bl ffff80001012f9c0 <__wake_up> 0.00 : ffff8000104b1010: b ffff8000104b0f68 <__sbq_wake_up+0x88> : __ll_sc__cmpxchg_case_mb_32(): 0.00 : ffff8000104b1014: and x0, x3, #0xffffffff 0.00 : ffff8000104b1018: add x5, x7, #0x24 0.00 : ffff8000104b101c: b ffff8000104b151c <__sbitmap_queue_get+0x284> 0.00 : ffff8000104b1020: b ffff8000104b0ffc <__sbq_wake_up+0x11c> Percent | Source code & Disassembly of vmlinux for cycles (482 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101257a0 : : find_busiest_group(): : * @env: The load balancing environment. : * : * Return: - The busiest group if imbalance exists. : */ : static struct sched_group *find_busiest_group(struct lb_env *env) : { 0.00 : ffff8000101257a0: stp x29, x30, [sp, #-400]! : init_sd_lb_stats(): : *sds = (struct sd_lb_stats){ 0.00 : ffff8000101257a4: mov w2, #0xffffffff // #-1 : find_busiest_group(): : { 0.00 : ffff8000101257a8: mov x29, sp : init_sd_lb_stats(): : *sds = (struct sd_lb_stats){ 0.00 : ffff8000101257ac: add x1, x29, #0x200 : find_busiest_group(): : { 0.00 : ffff8000101257b0: stp x19, x20, [sp, #16] 0.00 : ffff8000101257b4: stp x21, x22, [sp, #32] 0.00 : ffff8000101257b8: stp x23, x24, [sp, #48] 0.00 : ffff8000101257bc: stp x25, x26, [sp, #64] 0.00 : ffff8000101257c0: stp x27, x28, [sp, #80] 0.00 : ffff8000101257c4: mov x28, x0 0.00 : ffff8000101257c8: adrp x0, ffff800011909000 0.00 : ffff8000101257cc: add x0, x0, #0x908 : init_sd_lb_stats(): : *sds = (struct sd_lb_stats){ 0.00 : ffff8000101257d0: stp xzr, xzr, [x1, #-232] : find_busiest_group(): : { 0.00 : ffff8000101257d4: ldr x1, [x0] 0.00 : ffff8000101257d8: str x1, [x29, #392] 0.00 : ffff8000101257dc: mov x1, #0x0 // #0 : init_sd_lb_stats(): : *sds = (struct sd_lb_stats){ 0.00 : ffff8000101257e0: add x1, x29, #0x200 : update_sd_lb_stats(): : struct sched_domain *child = env->sd->child; 0.00 : ffff8000101257e4: ldr x0, [x28] : init_sd_lb_stats(): : *sds = (struct sd_lb_stats){ 0.00 : ffff8000101257e8: stp xzr, xzr, [x29, #200] 0.00 : ffff8000101257ec: stp xzr, xzr, [x29, #216] 0.00 : ffff8000101257f0: stp xzr, xzr, [x1, #-248] 0.00 : ffff8000101257f4: add x1, x29, #0x220 0.00 : ffff8000101257f8: stp xzr, xzr, [x29, #232] 0.00 : ffff8000101257fc: stp xzr, xzr, [x29, #248] 0.00 : ffff800010125800: stp xzr, xzr, [x1, #-248] 0.00 : ffff800010125804: stp xzr, xzr, [x1, #-232] 0.00 : ffff800010125808: stp xzr, xzr, [x1, #-216] 0.00 : ffff80001012580c: stp xzr, xzr, [x1, #-200] 0.18 : ffff800010125810: stp xzr, xzr, [x1, #-184] 0.00 : ffff800010125814: stp xzr, xzr, [x1, #-168] 0.00 : ffff800010125818: str w2, [x29, #288] : update_sd_lb_stats(): : if (env->idle == CPU_NEWLY_IDLE && READ_ONCE(nohz.has_blocked)) 0.00 : ffff80001012581c: ldr w1, [x28, #44] 0.00 : ffff800010125820: cmp w1, #0x2 : struct sched_domain *child = env->sd->child; 0.00 : ffff800010125824: ldr x1, [x0, #8] 0.00 : ffff800010125828: str x1, [x29, #104] : struct sched_group *sg = env->sd->groups; 0.00 : ffff80001012582c: ldr x23, [x0, #16] : if (env->idle == CPU_NEWLY_IDLE && READ_ONCE(nohz.has_blocked)) 0.00 : ffff800010125830: b.eq ffff800010125f68 // b.none : update_sg_lb_stats(): : struct rq *rq = cpu_rq(i); 0.00 : ffff800010125834: adrp x19, ffff800011528000 0.00 : ffff800010125838: add x19, x19, #0xe80 : find_busiest_group(): : { 0.00 : ffff80001012583c: mov w27, #0x0 // #0 : update_sd_lb_stats(): : local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 0.00 : ffff800010125840: ldr w1, [x28, #20] : cpumask_test_cpu(): : * : * Returns 1 if @cpu is set in @cpumask, else returns 0 : */ : static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask) : { : return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); 0.00 : ffff800010125844: add x22, x23, #0x20 : update_sd_lb_stats(): : struct sg_lb_stats *sgs = &tmp_sgs; 0.00 : ffff800010125848: add x26, x29, #0x80 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001012584c: add w2, w1, #0x3f 0.41 : ffff800010125850: cmp w1, #0x0 0.00 : ffff800010125854: csel w2, w2, w1, lt // lt = tstop 0.00 : ffff800010125858: asr w2, w2, #6 0.00 : ffff80001012585c: sxtw x2, w2 0.00 : ffff800010125860: ldr x2, [x22, x2, lsl #3] 0.00 : ffff800010125864: lsr x2, x2, x1 : update_sd_lb_stats(): : if (local_group) { 0.78 : ffff800010125868: and w3, w2, #0x1 10.64 : ffff80001012586c: str w3, [x29, #120] 0.00 : ffff800010125870: tbz w2, #0, ffff8000101258ac : if (env->idle != CPU_NEWLY_IDLE || 0.00 : ffff800010125874: ldr w2, [x28, #44] : sds->local = sg; 0.00 : ffff800010125878: str x23, [x29, #208] : if (env->idle != CPU_NEWLY_IDLE || 0.00 : ffff80001012587c: cmp w2, #0x2 0.23 : ffff800010125880: b.ne ffff8000101258a0 // b.any : time_after_eq(jiffies, sg->sgc->next_update)) 0.00 : ffff800010125884: adrp x2, ffff800011907000 0.00 : ffff800010125888: ldr x5, [x23, #16] : sgs = local; 0.00 : ffff80001012588c: add x26, x29, #0x140 : time_after_eq(jiffies, sg->sgc->next_update)) 0.00 : ffff800010125890: ldr x2, [x2, #2432] 0.00 : ffff800010125894: ldr x5, [x5, #32] 0.00 : ffff800010125898: sub x2, x2, x5 0.00 : ffff80001012589c: tbnz x2, #63, ffff8000101258ac : update_group_capacity(env->sd, env->dst_cpu); 0.00 : ffff8000101258a0: bl ffff800010125598 : sgs = local; 0.00 : ffff8000101258a4: add x26, x29, #0x140 0.00 : ffff8000101258a8: ldr w1, [x28, #20] : test_bit(): 0.25 : ffff8000101258ac: add w0, w1, #0x3f 0.00 : ffff8000101258b0: cmp w1, #0x0 0.50 : ffff8000101258b4: csel w0, w0, w1, lt // lt = tstop : update_sg_lb_stats(): : memset(sgs, 0, sizeof(*sgs)); 1.85 : ffff8000101258b8: stp xzr, xzr, [x26] 0.00 : ffff8000101258bc: stp xzr, xzr, [x26, #16] : struct rq *rq = cpu_rq(i); 0.00 : ffff8000101258c0: adrp x20, ffff800011909000 : test_bit(): 0.00 : ffff8000101258c4: asr w0, w0, #6 : update_sg_lb_stats(): : memset(sgs, 0, sizeof(*sgs)); 0.00 : ffff8000101258c8: stp xzr, xzr, [x26, #32] 0.00 : ffff8000101258cc: stp xzr, xzr, [x26, #48] : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff8000101258d0: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> : test_bit(): 0.00 : ffff8000101258d4: sxtw x0, w0 : update_sg_lb_stats(): : memset(sgs, 0, sizeof(*sgs)); 0.00 : ffff8000101258d8: str xzr, [x26, #64] : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff8000101258dc: add x24, x2, #0x2f0 : struct rq *rq = cpu_rq(i); 0.00 : ffff8000101258e0: add x20, x20, #0x928 : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff8000101258e4: mov w25, #0xffffffff // #-1 : test_bit(): 0.00 : ffff8000101258e8: ldr x21, [x22, x0, lsl #3] 0.00 : ffff8000101258ec: lsr x1, x21, x1 0.00 : ffff8000101258f0: and w0, w1, #0x1 0.47 : ffff8000101258f4: str w0, [x29, #124] : update_sg_lb_stats(): 0.63 : ffff8000101258f8: ldr x2, [x28, #56] 0.00 : ffff8000101258fc: mov w0, w25 0.00 : ffff800010125900: mov x1, x22 0.00 : ffff800010125904: bl ffff800010cb93f0 0.80 : ffff800010125908: mov w25, w0 0.77 : ffff80001012590c: ldr w0, [x24] 0.00 : ffff800010125910: cmp w25, w0 0.00 : ffff800010125914: b.cs ffff800010125a40 // b.hs, b.nlast : struct rq *rq = cpu_rq(i); 0.00 : ffff800010125918: sxtw x8, w25 : if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false)) 3.31 : ffff80001012591c: ldr w0, [x28, #64] : struct rq *rq = cpu_rq(i); 0.00 : ffff800010125920: mov x2, x19 0.00 : ffff800010125924: ldr x5, [x20, x8, lsl #3] 0.00 : ffff800010125928: add x21, x2, x5 : if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false)) 0.17 : ffff80001012592c: tbnz w0, #4, ffff800010125c24 : sgs->group_load += cpu_load(rq); 2.54 : ffff800010125930: ldr x8, [x21, #288] : cpu_util(): : cfs_rq = &cpu_rq(cpu)->cfs; 0.03 : ffff800010125934: mov x0, x19 : update_sg_lb_stats(): : sgs->group_load += cpu_load(rq); 22.82 : ffff800010125938: ldr x1, [x26, #8] : *sg_status |= SG_OVERLOAD; 0.00 : ffff80001012593c: orr w12, w27, #0x1 : cpu_util(): : cfs_rq = &cpu_rq(cpu)->cfs; 0.00 : ffff800010125940: add x0, x0, x5 : update_sg_lb_stats(): : sgs->group_load += cpu_load(rq); 0.00 : ffff800010125944: add x1, x1, x8 0.00 : ffff800010125948: str x1, [x26, #8] : sgs->group_util += cpu_util(i); 0.81 : ffff80001012594c: ldr x13, [x26, #24] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010125950: ldr x1, [x0, #304] 1.44 : ffff800010125954: ldr w14, [x0, #312] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.17 : ffff800010125958: ldr x5, [x0, #2488] : util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); 0.42 : ffff80001012595c: cmp w14, w1 : update_sg_lb_stats(): : sgs->sum_h_nr_running += rq->cfs.h_nr_running; 9.31 : ffff800010125960: ldr w8, [x26, #36] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff800010125964: csel w1, w1, w14, ls // ls = plast : update_sg_lb_stats(): : sgs->sum_nr_running += nr_running; 0.00 : ffff800010125968: ldr w9, [x26, #32] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff80001012596c: cmp x5, x1 0.00 : ffff800010125970: csel x1, x5, x1, ls // ls = plast : update_sg_lb_stats(): : sgs->group_util += cpu_util(i); 0.00 : ffff800010125974: add x1, x13, x1 0.00 : ffff800010125978: str x1, [x26, #24] : sgs->nr_preferred_running += rq->nr_preferred_running; 0.25 : ffff80001012597c: ldp w11, w10, [x26, #64] : sgs->sum_h_nr_running += rq->cfs.h_nr_running; 0.38 : ffff800010125980: ldr w1, [x21, #156] 0.00 : ffff800010125984: add w1, w8, w1 3.16 : ffff800010125988: str w1, [x26, #36] : nr_running = rq->nr_running; 0.00 : ffff80001012598c: ldr w8, [x21, #4] : sgs->sum_nr_running += nr_running; 0.22 : ffff800010125990: add w1, w9, w8 0.85 : ffff800010125994: str w1, [x26, #32] : *sg_status |= SG_OVERLOAD; 0.23 : ffff800010125998: cmp w8, #0x1 : __read_once_size(): 0.95 : ffff80001012599c: ldr x5, [x0, #304] : update_sg_lb_stats(): 0.00 : ffff8000101259a0: csel w27, w12, w27, gt : __read_once_size(): 1.20 : ffff8000101259a4: ldr w9, [x0, #312] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff8000101259a8: ldr x1, [x0, #2488] : update_sg_lb_stats(): : sgs->nr_numa_running += rq->nr_numa_running; 0.95 : ffff8000101259ac: ldr w12, [x21, #8] : cpu_util(): : util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); 0.00 : ffff8000101259b0: cmp w9, w5 : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff8000101259b4: csel w5, w5, w9, ls // ls = plast : cpu_overutilized(): : return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); 0.00 : ffff8000101259b8: ldr x9, [x0, #2480] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff8000101259bc: cmp x1, x5 : update_sg_lb_stats(): : sgs->nr_numa_running += rq->nr_numa_running; 0.00 : ffff8000101259c0: add w0, w11, w12 0.00 : ffff8000101259c4: str w0, [x26, #64] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff8000101259c8: csel x0, x1, x5, ls // ls = plast : cpu_overutilized(): : return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); 0.00 : ffff8000101259cc: lsl x1, x9, #10 : update_sg_lb_stats(): : *sg_status |= SG_OVERUTILIZED; 0.00 : ffff8000101259d0: orr w9, w27, #0x2 : sgs->nr_preferred_running += rq->nr_preferred_running; 0.20 : ffff8000101259d4: ldr w5, [x21, #12] : cpu_overutilized(): : return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); 0.00 : ffff8000101259d8: add x0, x0, x0, lsl #2 : update_sg_lb_stats(): : sgs->nr_preferred_running += rq->nr_preferred_running; 0.00 : ffff8000101259dc: add w5, w10, w5 2.80 : ffff8000101259e0: str w5, [x26, #68] : *sg_status |= SG_OVERUTILIZED; 0.00 : ffff8000101259e4: cmp x1, x0, lsl #8 0.00 : ffff8000101259e8: csel w27, w9, w27, ls // ls = plast : if (!nr_running && idle_cpu(i)) { 0.00 : ffff8000101259ec: cbz w8, ffff800010125c08 : if (local_group) 0.00 : ffff8000101259f0: ldr w0, [x29, #124] 0.00 : ffff8000101259f4: cbnz w0, ffff8000101258f8 : if (env->sd->flags & SD_ASYM_CPUCAPACITY && 0.00 : ffff8000101259f8: ldr x0, [x28] 0.24 : ffff8000101259fc: ldr w0, [x0, #56] 0.23 : ffff800010125a00: tbz w0, #6, ffff8000101258f8 : sgs->group_misfit_task_load < rq->misfit_task_load) { 0.00 : ffff800010125a04: ldr x0, [x21, #2512] : if (env->sd->flags & SD_ASYM_CPUCAPACITY && 0.00 : ffff800010125a08: ldr x1, [x26, #56] 0.00 : ffff800010125a0c: cmp x1, x0 0.00 : ffff800010125a10: b.cs ffff8000101258f8 // b.hs, b.nlast : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff800010125a14: ldr x2, [x28, #56] 0.00 : ffff800010125a18: mov x1, x22 : sgs->group_misfit_task_load = rq->misfit_task_load; 0.00 : ffff800010125a1c: str x0, [x26, #56] : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff800010125a20: mov w0, w25 : *sg_status |= SG_OVERLOAD; 0.00 : ffff800010125a24: orr w27, w27, #0x1 : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff800010125a28: bl ffff800010cb93f0 0.00 : ffff800010125a2c: mov w25, w0 0.00 : ffff800010125a30: ldr w0, [x24] 0.00 : ffff800010125a34: cmp w25, w0 0.00 : ffff800010125a38: b.cc ffff800010125918 // b.lo, b.ul, b.last 0.00 : ffff800010125a3c: nop : if (env->sd->flags & SD_ASYM_PACKING && 0.00 : ffff800010125a40: ldr x0, [x28] 0.00 : ffff800010125a44: mov x10, x0 0.20 : ffff800010125a48: ldr w1, [x0, #56] 0.25 : ffff800010125a4c: tbz w1, #11, ffff800010125a64 0.00 : ffff800010125a50: ldr w1, [x28, #44] 0.00 : ffff800010125a54: cmp w1, #0x1 0.00 : ffff800010125a58: b.eq ffff800010125a64 // b.none : env->idle != CPU_NOT_IDLE && 0.00 : ffff800010125a5c: ldr w1, [x26, #36] 0.00 : ffff800010125a60: cbnz w1, ffff800010125dd0 : sgs->group_capacity = group->sgc->capacity; 1.63 : ffff800010125a64: ldr x1, [x23, #16] : group_is_overloaded(): : if (sgs->sum_nr_running <= sgs->group_weight) 0.22 : ffff800010125a68: ldr w6, [x26, #32] : update_sg_lb_stats(): : sgs->group_capacity = group->sgc->capacity; 0.25 : ffff800010125a6c: ldr x1, [x1, #8] 15.83 : ffff800010125a70: str x1, [x26, #16] : sgs->group_weight = group->group_weight; 0.22 : ffff800010125a74: ldr w2, [x23, #12] 0.21 : ffff800010125a78: str w2, [x26, #44] : group_is_overloaded(): : if (sgs->sum_nr_running <= sgs->group_weight) 0.00 : ffff800010125a7c: cmp w2, w6 : update_sg_lb_stats(): : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 0.00 : ffff800010125a80: ldr w8, [x0, #44] : group_is_overloaded(): : if (sgs->sum_nr_running <= sgs->group_weight) 0.00 : ffff800010125a84: b.cs ffff800010125aa4 // b.hs, b.nlast : (sgs->group_util * imbalance_pct)) 0.00 : ffff800010125a88: ldr x11, [x26, #24] 0.00 : ffff800010125a8c: mov w5, w8 : if ((sgs->group_capacity * 100) < 0.00 : ffff800010125a90: add x9, x1, x1, lsl #1 0.00 : ffff800010125a94: add x9, x1, x9, lsl #3 : (sgs->group_util * imbalance_pct)) 0.00 : ffff800010125a98: mul x5, x5, x11 : if ((sgs->group_capacity * 100) < 0.00 : ffff800010125a9c: cmp x5, x9, lsl #2 0.00 : ffff800010125aa0: b.hi ffff800010125c54 // b.pmore : sg_imbalanced(): : return group->sgc->imbalance; 0.91 : ffff800010125aa4: ldr x5, [x23, #16] : group_classify(): : if (sg_imbalanced(group)) 0.20 : ffff800010125aa8: ldr w5, [x5, #40] 0.00 : ffff800010125aac: cbnz w5, ffff800010125cc4 : if (sgs->group_asym_packing) 4.09 : ffff800010125ab0: ldr w5, [x26, #52] 0.00 : ffff800010125ab4: cbnz w5, ffff800010125d00 : if (sgs->group_misfit_task_load) 0.23 : ffff800010125ab8: ldr x5, [x26, #56] 0.00 : ffff800010125abc: cbnz x5, ffff800010125d64 : group_has_capacity(): : if (sgs->sum_nr_running < sgs->group_weight) 0.00 : ffff800010125ac0: cmp w2, w6 0.00 : ffff800010125ac4: b.hi ffff800010125ae4 // b.pmore : (sgs->group_util * imbalance_pct)) 0.00 : ffff800010125ac8: ldr x5, [x26, #24] 0.00 : ffff800010125acc: mov w8, w8 : if ((sgs->group_capacity * 100) > 0.00 : ffff800010125ad0: add x2, x1, x1, lsl #1 0.00 : ffff800010125ad4: add x2, x1, x2, lsl #3 : (sgs->group_util * imbalance_pct)) 0.00 : ffff800010125ad8: mul x1, x8, x5 : if ((sgs->group_capacity * 100) > 0.00 : ffff800010125adc: cmp x1, x2, lsl #2 0.00 : ffff800010125ae0: b.cs ffff800010125f20 // b.hs, b.nlast : update_sd_lb_stats(): : if (local_group) 0.00 : ffff800010125ae4: ldr w1, [x29, #120] : update_sg_lb_stats(): : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 0.00 : ffff800010125ae8: str wzr, [x26, #48] : update_sd_lb_stats(): : if (local_group) 0.00 : ffff800010125aec: cbz w1, ffff800010125e28 0.00 : ffff800010125af0: ldr x5, [x26, #8] : sds->total_capacity += sgs->group_capacity; 0.00 : ffff800010125af4: ldp x2, x1, [x29, #216] : sds->total_load += sgs->group_load; 0.00 : ffff800010125af8: add x2, x2, x5 2.06 : ffff800010125afc: str x2, [x29, #216] : sds->total_capacity += sgs->group_capacity; 0.00 : ffff800010125b00: ldr x2, [x26, #16] 0.00 : ffff800010125b04: add x1, x1, x2 0.43 : ffff800010125b08: str x1, [x29, #224] : } while (sg != env->sd->groups); 0.00 : ffff800010125b0c: ldr x1, [x0, #16] : sg = sg->next; 0.41 : ffff800010125b10: ldr x23, [x23] : } while (sg != env->sd->groups); 0.00 : ffff800010125b14: cmp x23, x1 0.00 : ffff800010125b18: b.ne ffff800010125840 // b.any : sds->prefer_sibling = child && child->flags & SD_PREFER_SIBLING; 0.00 : ffff800010125b1c: ldr x2, [x29, #104] 0.00 : ffff800010125b20: mov w1, #0x0 // #0 0.00 : ffff800010125b24: cbz x2, ffff800010125b30 0.00 : ffff800010125b28: ldr w1, [x2, #56] 0.00 : ffff800010125b2c: ubfx x1, x1, #12, #1 : if ((env->flags & LBF_NOHZ_AGAIN) && 0.00 : ffff800010125b30: ldr w2, [x28, #64] : sds->prefer_sibling = child && child->flags & SD_PREFER_SIBLING; 0.22 : ffff800010125b34: str w1, [x29, #240] : if ((env->flags & LBF_NOHZ_AGAIN) && 0.00 : ffff800010125b38: tbnz w2, #5, ffff800010125f88 : if (env->sd->flags & SD_NUMA) 0.00 : ffff800010125b3c: ldr w0, [x10, #56] 0.00 : ffff800010125b40: tbz w0, #14, ffff800010125b5c : env->fbq_type = fbq_classify_group(&sds->busiest_stat); 0.00 : ffff800010125b44: ldr w1, [x29, #284] : fbq_classify_group(): : return regular; 0.00 : ffff800010125b48: mov w0, #0x0 // #0 : if (sgs->sum_h_nr_running > sgs->nr_numa_running) 0.00 : ffff800010125b4c: ldr w2, [x29, #312] 0.00 : ffff800010125b50: cmp w1, w2 0.00 : ffff800010125b54: b.ls ffff800010125f0c // b.plast : update_sd_lb_stats(): : env->fbq_type = fbq_classify_group(&sds->busiest_stat); 0.00 : ffff800010125b58: str w0, [x28, #80] : if (!env->sd->parent) { 0.00 : ffff800010125b5c: ldr x1, [x10] 0.00 : ffff800010125b60: and w0, w27, #0x2 0.00 : ffff800010125b64: cbz x1, ffff800010126000 : } else if (sg_status & SG_OVERUTILIZED) { 0.00 : ffff800010125b68: cbz w0, ffff800010125b7c : struct root_domain *rd = env->dst_rq->rd; 0.00 : ffff800010125b6c: ldr x0, [x28, #24] : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010125b70: mov w1, #0x2 // #2 : update_sd_lb_stats(): 0.00 : ffff800010125b74: ldr x0, [x0, #2464] : __write_once_size(): 0.20 : ffff800010125b78: str w1, [x0, #92] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010125b7c: nop : find_busiest_group(): : : local = &sds.local_stat; : busiest = &sds.busiest_stat; : : /* There is no busy sibling group to pull tasks from */ : if (!sds.busiest) 0.00 : ffff800010125b80: ldr x0, [x29, #200] 0.00 : ffff800010125b84: mov x3, x0 0.00 : ffff800010125b88: cbz x0, ffff800010125e1c : goto out_balanced; : : /* Misfit tasks should be dealt with regardless of the avg load */ : if (busiest->group_type == group_misfit_task) 0.00 : ffff800010125b8c: ldr w1, [x29, #296] 0.00 : ffff800010125b90: cmp w1, #0x2 0.00 : ffff800010125b94: b.eq ffff800010125fbc // b.none : /* : * If the busiest group is imbalanced the below checks don't : * work because they assume all things are equal, which typically : * isn't true due to cpus_ptr constraints and the like. : */ : if (busiest->group_type == group_imbalanced) 0.00 : ffff800010125b98: sub w2, w1, #0x3 0.00 : ffff800010125b9c: cmp w2, #0x1 0.00 : ffff800010125ba0: b.hi ffff800010125e8c // b.pmore : calculate_imbalance(): : if (busiest->group_type == group_asym_packing) { 0.00 : ffff800010125ba4: cmp w1, #0x3 0.00 : ffff800010125ba8: b.eq ffff8000101260ec // b.none : if (busiest->group_type == group_imbalanced) { 0.00 : ffff800010125bac: cmp w1, #0x4 0.00 : ffff800010125bb0: b.eq ffff800010125ef8 // b.none 0.00 : ffff800010125bb4: ldr w2, [x29, #368] : if (local->group_type == group_has_spare) { 0.00 : ffff800010125bb8: cbz w2, ffff800010126034 : if (local->group_type < group_overloaded) { 0.00 : ffff800010125bbc: cmp w2, #0x4 0.00 : ffff800010125bc0: ldr x4, [x29, #336] 0.00 : ffff800010125bc4: b.ls ffff800010126148 // b.plast 0.00 : ffff800010125bc8: ldr x2, [x29, #232] 0.00 : ffff800010125bcc: ldr x1, [x29, #320] : env->imbalance = min( 0.00 : ffff800010125bd0: ldr x0, [x29, #248] 0.00 : ffff800010125bd4: sub x1, x2, x1 0.00 : ffff800010125bd8: ldr x5, [x29, #264] 0.00 : ffff800010125bdc: sub x0, x0, x2 : env->migration_type = migrate_load; 0.00 : ffff800010125be0: str wzr, [x28, #84] : env->imbalance = min( 0.00 : ffff800010125be4: mul x1, x1, x4 0.00 : ffff800010125be8: mul x0, x0, x5 0.00 : ffff800010125bec: cmp x0, x1 0.00 : ffff800010125bf0: csel x0, x0, x1, ls // ls = plast : ) / SCHED_CAPACITY_SCALE; 0.00 : ffff800010125bf4: lsr x0, x0, #10 : env->imbalance = min( 0.00 : ffff800010125bf8: str x0, [x28, #48] 0.00 : ffff800010125bfc: cmp x0, #0x0 0.00 : ffff800010125c00: csel x0, x3, xzr, ne // ne = any : find_busiest_group(): 0.00 : ffff800010125c04: b ffff800010125fcc : update_sg_lb_stats(): : if (!nr_running && idle_cpu(i)) { 0.00 : ffff800010125c08: mov w0, w25 0.00 : ffff800010125c0c: bl ffff800010118820 0.42 : ffff800010125c10: cbz w0, ffff8000101259f0 : sgs->idle_cpus++; 0.00 : ffff800010125c14: ldr w0, [x26, #40] 0.00 : ffff800010125c18: add w0, w0, #0x1 0.00 : ffff800010125c1c: str w0, [x26, #40] 0.00 : ffff800010125c20: b ffff8000101258f8 0.22 : ffff800010125c24: str x8, [x29, #112] : if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false)) 0.00 : ffff800010125c28: mov w1, #0x0 // #0 0.00 : ffff800010125c2c: mov x0, x21 0.00 : ffff800010125c30: bl ffff800010122dd0 0.00 : ffff800010125c34: ldr x8, [x29, #112] 0.00 : ffff800010125c38: tst w0, #0xff 0.00 : ffff800010125c3c: b.eq ffff800010125c4c // b.none : env->flags |= LBF_NOHZ_AGAIN; 0.00 : ffff800010125c40: ldr w0, [x28, #64] 0.00 : ffff800010125c44: orr w0, w0, #0x20 0.00 : ffff800010125c48: str w0, [x28, #64] 0.24 : ffff800010125c4c: ldr x5, [x20, x8, lsl #3] 0.00 : ffff800010125c50: b ffff800010125930 : sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010125c54: ldr x5, [x26, #8] : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 0.00 : ffff800010125c58: mov w2, #0x5 // #5 0.00 : ffff800010125c5c: str w2, [x26, #48] : sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010125c60: mov x6, x5 0.00 : ffff800010125c64: lsl x2, x5, #10 0.00 : ffff800010125c68: udiv x1, x2, x1 : update_sd_lb_stats(): : if (local_group) 0.00 : ffff800010125c6c: ldr w2, [x29, #120] : update_sg_lb_stats(): : sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010125c70: str x1, [x26] : update_sd_lb_stats(): : if (local_group) 0.00 : ffff800010125c74: cbnz w2, ffff800010125af4 : update_sd_pick_busiest(): : if (!sgs->sum_h_nr_running) 0.00 : ffff800010125c78: ldr w2, [x26, #36] 0.00 : ffff800010125c7c: cbz w2, ffff800010125af4 : if (sgs->group_type > busiest->group_type) 0.00 : ffff800010125c80: ldr w2, [x29, #296] 0.00 : ffff800010125c84: cmp w2, #0x4 0.00 : ffff800010125c88: b.hi ffff800010126018 // b.pmore : update_sd_lb_stats(): : sds->busiest = sg; 0.00 : ffff800010125c8c: str x23, [x29, #200] : sds->busiest_stat = *sgs; 0.00 : ffff800010125c90: add x1, x29, #0x200 0.00 : ffff800010125c94: ldp x8, x9, [x26] 0.00 : ffff800010125c98: stp x8, x9, [x29, #248] 0.00 : ffff800010125c9c: ldp x8, x9, [x26, #16] 0.00 : ffff800010125ca0: stp x8, x9, [x1, #-248] 0.00 : ffff800010125ca4: ldp x8, x9, [x26, #32] 0.00 : ffff800010125ca8: stp x8, x9, [x1, #-232] 0.00 : ffff800010125cac: add x1, x29, #0x220 0.00 : ffff800010125cb0: ldp x8, x9, [x26, #48] 0.00 : ffff800010125cb4: stp x8, x9, [x1, #-248] 0.00 : ffff800010125cb8: ldr x1, [x26, #64] 0.00 : ffff800010125cbc: str x1, [x29, #312] 0.00 : ffff800010125cc0: b ffff800010125af0 : update_sg_lb_stats(): : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 0.00 : ffff800010125cc4: mov w1, #0x4 // #4 0.00 : ffff800010125cc8: str w1, [x26, #48] : update_sd_lb_stats(): : if (local_group) 0.00 : ffff800010125ccc: ldr w1, [x29, #120] 0.00 : ffff800010125cd0: cbnz w1, ffff800010125af0 : update_sd_pick_busiest(): : if (!sgs->sum_h_nr_running) 0.00 : ffff800010125cd4: ldr w1, [x26, #36] 0.00 : ffff800010125cd8: cbz w1, ffff800010125af0 : if (sgs->group_type > busiest->group_type) 0.00 : ffff800010125cdc: ldr w1, [x29, #296] 0.00 : ffff800010125ce0: cmp w1, #0x3 0.00 : ffff800010125ce4: b.ls ffff800010125c8c // b.plast : if (sgs->group_type < busiest->group_type) 0.00 : ffff800010125ce8: cmp w1, #0x4 0.00 : ffff800010125cec: b.eq ffff800010125d58 // b.none 0.23 : ffff800010125cf0: ldr x6, [x26, #8] 0.00 : ffff800010125cf4: mov x5, x6 0.00 : ffff800010125cf8: mov x10, x0 0.00 : ffff800010125cfc: b ffff800010125af4 : update_sg_lb_stats(): : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 0.00 : ffff800010125d00: mov w1, #0x3 // #3 0.00 : ffff800010125d04: str w1, [x26, #48] : update_sd_lb_stats(): : if (local_group) 0.00 : ffff800010125d08: ldr w1, [x29, #120] 0.00 : ffff800010125d0c: cbnz w1, ffff800010125af0 : update_sd_pick_busiest(): : if (!sgs->sum_h_nr_running) 0.00 : ffff800010125d10: ldr w1, [x26, #36] 0.00 : ffff800010125d14: cbz w1, ffff800010125af0 : if (sgs->group_type > busiest->group_type) 0.00 : ffff800010125d18: ldr w1, [x29, #296] 0.00 : ffff800010125d1c: cmp w1, #0x2 0.00 : ffff800010125d20: b.ls ffff800010125c8c // b.plast : if (sgs->group_type < busiest->group_type) 0.00 : ffff800010125d24: cmp w1, #0x3 0.00 : ffff800010125d28: b.ne ffff800010125cf0 // b.any : if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu)) 0.00 : ffff800010125d2c: ldr x1, [x29, #200] : sched_asym_prefer(): : return scale_load_down(se->runnable_weight); : } : : static inline bool sched_asym_prefer(int a, int b) : { : return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b); 0.00 : ffff800010125d30: ldr w0, [x23, #24] : update_sd_pick_busiest(): 0.00 : ffff800010125d34: ldr w21, [x1, #24] : sched_asym_prefer(): 0.00 : ffff800010125d38: bl ffff800010123c30 0.00 : ffff800010125d3c: mov w20, w0 0.00 : ffff800010125d40: mov w0, w21 0.00 : ffff800010125d44: bl ffff800010123c30 : update_sd_pick_busiest(): 0.00 : ffff800010125d48: cmp w20, w0 0.00 : ffff800010125d4c: ldr x0, [x28] 0.00 : ffff800010125d50: mov x10, x0 0.00 : ffff800010125d54: b.le ffff800010125e50 0.00 : ffff800010125d58: mov x10, x0 0.00 : ffff800010125d5c: ldr x5, [x26, #8] 0.00 : ffff800010125d60: b ffff800010125af4 : update_sg_lb_stats(): : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 0.00 : ffff800010125d64: mov w1, #0x2 // #2 0.00 : ffff800010125d68: str w1, [x26, #48] : update_sd_lb_stats(): : if (local_group) 0.00 : ffff800010125d6c: ldr w1, [x29, #120] 0.00 : ffff800010125d70: cbnz w1, ffff800010125af0 : update_sd_pick_busiest(): : if (!sgs->sum_h_nr_running) 0.00 : ffff800010125d74: ldr w1, [x26, #36] 0.00 : ffff800010125d78: cbz w1, ffff800010125af0 : (!group_smaller_max_cpu_capacity(sg, sds->local) || 0.00 : ffff800010125d7c: ldr x2, [x29, #208] : group_smaller_max_cpu_capacity(): : return fits_capacity(sg->sgc->max_capacity, ref->sgc->max_capacity); 0.00 : ffff800010125d80: ldr x1, [x23, #16] 0.00 : ffff800010125d84: ldr x2, [x2, #16] 0.00 : ffff800010125d88: ldr x1, [x1, #24] 0.00 : ffff800010125d8c: ldr x2, [x2, #24] 0.00 : ffff800010125d90: add x1, x1, x1, lsl #2 0.00 : ffff800010125d94: lsl x2, x2, #10 : update_sd_pick_busiest(): : if (sgs->group_type == group_misfit_task && 0.00 : ffff800010125d98: cmp x2, x1, lsl #8 0.00 : ffff800010125d9c: b.ls ffff800010125af0 // b.plast : (!group_smaller_max_cpu_capacity(sg, sds->local) || 0.00 : ffff800010125da0: ldr w1, [x29, #368] 0.00 : ffff800010125da4: cbnz w1, ffff800010125af0 : if (sgs->group_type > busiest->group_type) 0.00 : ffff800010125da8: ldr w1, [x29, #296] 0.00 : ffff800010125dac: cmp w1, #0x1 0.00 : ffff800010125db0: b.ls ffff800010125c8c // b.plast : if (sgs->group_type < busiest->group_type) 0.00 : ffff800010125db4: cmp w1, #0x2 0.00 : ffff800010125db8: b.ne ffff800010125cf0 // b.any : if (sgs->group_misfit_task_load < busiest->group_misfit_task_load) 0.00 : ffff800010125dbc: ldr x1, [x29, #304] 0.00 : ffff800010125dc0: mov x10, x0 0.00 : ffff800010125dc4: cmp x5, x1 0.00 : ffff800010125dc8: b.cs ffff800010125c8c // b.hs, b.nlast 0.00 : ffff800010125dcc: b ffff800010125af0 : sched_asym_prefer(): 0.00 : ffff800010125dd0: ldr w0, [x28, #20] : update_sg_lb_stats(): : sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu)) { 0.00 : ffff800010125dd4: ldr w21, [x23, #24] : sched_asym_prefer(): 0.00 : ffff800010125dd8: bl ffff800010123c30 0.00 : ffff800010125ddc: mov w20, w0 0.00 : ffff800010125de0: mov w0, w21 0.00 : ffff800010125de4: bl ffff800010123c30 : update_sg_lb_stats(): : sgs->sum_h_nr_running && 0.00 : ffff800010125de8: cmp w20, w0 0.00 : ffff800010125dec: ldr x0, [x28] 0.00 : ffff800010125df0: b.le ffff800010125dfc : sgs->group_asym_packing = 1; 0.00 : ffff800010125df4: mov w1, #0x1 // #1 0.00 : ffff800010125df8: str w1, [x26, #52] 0.00 : ffff800010125dfc: mov x10, x0 0.00 : ffff800010125e00: b ffff800010125a64 : find_busiest_group(): : struct root_domain *rd = env->dst_rq->rd; 0.00 : ffff800010125e04: ldr x0, [x28, #24] 0.00 : ffff800010125e08: ldr x0, [x0, #2464] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010125e0c: ldr x1, [x0, #4384] : find_busiest_group(): : if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized)) 0.00 : ffff800010125e10: cbz x1, ffff800010125b80 : __read_once_size(): 0.00 : ffff800010125e14: ldr w0, [x0, #92] : find_busiest_group(): 0.00 : ffff800010125e18: cbnz w0, ffff800010125b80 : calculate_imbalance(env, &sds); : return env->imbalance ? sds.busiest : NULL; : : out_balanced: : env->imbalance = 0; : return NULL; 0.18 : ffff800010125e1c: mov x0, #0x0 // #0 : env->imbalance = 0; 0.00 : ffff800010125e20: str xzr, [x28, #48] : return NULL; 0.00 : ffff800010125e24: b ffff800010125fcc : update_sd_pick_busiest(): : if (!sgs->sum_h_nr_running) 0.58 : ffff800010125e28: ldr w1, [x26, #36] 0.00 : ffff800010125e2c: cbz w1, ffff800010125af0 : if (sgs->group_type < busiest->group_type) 0.00 : ffff800010125e30: ldr w1, [x29, #296] 0.00 : ffff800010125e34: cbnz w1, ffff800010125cf0 : if (sgs->idle_cpus > busiest->idle_cpus) 0.00 : ffff800010125e38: ldr w2, [x26, #40] 0.00 : ffff800010125e3c: mov x10, x0 0.00 : ffff800010125e40: ldr w1, [x29, #288] 0.00 : ffff800010125e44: cmp w2, w1 0.00 : ffff800010125e48: b.hi ffff800010125af0 // b.pmore : else if ((sgs->idle_cpus == busiest->idle_cpus) && 0.00 : ffff800010125e4c: b.eq ffff800010126194 // b.none : if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 0.00 : ffff800010125e50: ldr w1, [x0, #56] 0.00 : ffff800010125e54: tbz w1, #6, ffff800010125c8c 0.00 : ffff800010125e58: ldr w1, [x26, #48] 0.00 : ffff800010125e5c: cmp w1, #0x1 0.00 : ffff800010125e60: b.hi ffff800010125c8c // b.pmore : (group_smaller_min_cpu_capacity(sds->local, sg))) 0.00 : ffff800010125e64: ldr x1, [x29, #208] : group_smaller_min_cpu_capacity(): : return fits_capacity(sg->sgc->min_capacity, ref->sgc->min_capacity); 0.00 : ffff800010125e68: ldr x2, [x23, #16] 0.00 : ffff800010125e6c: ldr x1, [x1, #16] 0.00 : ffff800010125e70: ldr x2, [x2, #16] 0.00 : ffff800010125e74: ldr x1, [x1, #16] 0.00 : ffff800010125e78: lsl x2, x2, #10 0.00 : ffff800010125e7c: add x1, x1, x1, lsl #2 : update_sd_pick_busiest(): : (sgs->group_type <= group_fully_busy) && 0.00 : ffff800010125e80: cmp x2, x1, lsl #8 0.00 : ffff800010125e84: b.ls ffff800010125c8c // b.plast 0.00 : ffff800010125e88: b ffff800010125af0 : find_busiest_group(): : if (local->group_type > busiest->group_type) 0.00 : ffff800010125e8c: ldr w2, [x29, #368] 0.00 : ffff800010125e90: cmp w1, w2 0.00 : ffff800010125e94: b.cc ffff800010125e1c // b.lo, b.ul, b.last : if (local->group_type == group_overloaded) { 0.18 : ffff800010125e98: cmp w2, #0x5 0.00 : ffff800010125e9c: b.eq ffff800010126100 // b.none : if (sds.prefer_sibling && local->group_type == group_has_spare && 0.00 : ffff800010125ea0: ldr w4, [x29, #240] 0.00 : ffff800010125ea4: cmp w4, #0x0 0.00 : ffff800010125ea8: ccmp w2, #0x0, #0x0, ne // ne = any 0.00 : ffff800010125eac: b.eq ffff8000101260d4 // b.none : if (busiest->group_type != group_overloaded) { 0.00 : ffff800010125eb0: cmp w1, #0x5 0.00 : ffff800010125eb4: b.eq ffff80001012608c // b.none : if (env->idle == CPU_NOT_IDLE) 0.00 : ffff800010125eb8: ldr w2, [x28, #44] 0.00 : ffff800010125ebc: cmp w2, #0x1 0.00 : ffff800010125ec0: b.eq ffff800010125e1c // b.none : if (busiest->group_weight > 1 && 0.00 : ffff800010125ec4: ldr w2, [x29, #292] 0.00 : ffff800010125ec8: cmp w2, #0x1 0.00 : ffff800010125ecc: b.ls ffff800010125ee4 // b.plast : local->idle_cpus <= (busiest->idle_cpus + 1)) 0.00 : ffff800010125ed0: ldr w2, [x29, #288] : if (busiest->group_weight > 1 && 0.00 : ffff800010125ed4: ldr w4, [x29, #360] : local->idle_cpus <= (busiest->idle_cpus + 1)) 0.00 : ffff800010125ed8: add w2, w2, #0x1 : if (busiest->group_weight > 1 && 0.00 : ffff800010125edc: cmp w4, w2 0.00 : ffff800010125ee0: b.ls ffff800010125e1c // b.plast : if (busiest->sum_h_nr_running == 1) 0.00 : ffff800010125ee4: ldr w2, [x29, #284] 0.00 : ffff800010125ee8: cmp w2, #0x1 0.00 : ffff800010125eec: b.eq ffff800010125e1c // b.none : calculate_imbalance(): : if (busiest->group_type == group_imbalanced) { 0.00 : ffff800010125ef0: cmp w1, #0x4 0.00 : ffff800010125ef4: b.ne ffff800010125bb4 // b.any : env->migration_type = migrate_task; 0.00 : ffff800010125ef8: mov w1, #0x2 // #2 : env->imbalance = 1; 0.00 : ffff800010125efc: mov x2, #0x1 // #1 0.00 : ffff800010125f00: str x2, [x28, #48] : env->migration_type = migrate_task; 0.00 : ffff800010125f04: str w1, [x28, #84] 0.00 : ffff800010125f08: b ffff800010125fcc : fbq_classify_group(): : if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 0.00 : ffff800010125f0c: ldr w0, [x29, #316] : return remote; 0.00 : ffff800010125f10: cmp w1, w0 0.00 : ffff800010125f14: cset w0, ls // ls = plast 0.00 : ffff800010125f18: add w0, w0, #0x1 0.00 : ffff800010125f1c: b ffff800010125b58 : update_sg_lb_stats(): : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 0.00 : ffff800010125f20: mov w1, #0x1 // #1 0.00 : ffff800010125f24: str w1, [x26, #48] : update_sd_lb_stats(): : if (local_group) 0.00 : ffff800010125f28: ldr w1, [x29, #120] 0.00 : ffff800010125f2c: cbnz w1, ffff800010125af0 : update_sd_pick_busiest(): : if (!sgs->sum_h_nr_running) 0.00 : ffff800010125f30: ldr w1, [x26, #36] 0.00 : ffff800010125f34: cbz w1, ffff800010125af0 : if (sgs->group_type > busiest->group_type) 0.00 : ffff800010125f38: ldr w1, [x29, #296] 0.00 : ffff800010125f3c: cbz w1, ffff800010125c8c : if (sgs->group_type < busiest->group_type) 0.00 : ffff800010125f40: cmp w1, #0x1 0.00 : ffff800010125f44: b.hi ffff800010125cf0 // b.pmore : if (sgs->avg_load <= busiest->avg_load) 0.00 : ffff800010125f48: ldr x2, [x26] 0.00 : ffff800010125f4c: mov x10, x0 0.00 : ffff800010125f50: ldr x1, [x29, #248] 0.00 : ffff800010125f54: cmp x2, x1 0.00 : ffff800010125f58: b.ls ffff800010125af0 // b.plast : if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 0.00 : ffff800010125f5c: ldr w1, [x0, #56] 0.00 : ffff800010125f60: tbnz w1, #6, ffff800010125e64 0.00 : ffff800010125f64: b ffff800010125c8c : __read_once_size(): 0.00 : ffff800010125f68: adrp x1, ffff800011afb000 0.00 : ffff800010125f6c: add x1, x1, #0xfc0 0.00 : ffff800010125f70: ldr w1, [x1, #36] : update_sd_lb_stats(): : if (env->idle == CPU_NEWLY_IDLE && READ_ONCE(nohz.has_blocked)) 0.00 : ffff800010125f74: cbz w1, ffff800010125834 : env->flags |= LBF_NOHZ_STATS; 0.00 : ffff800010125f78: ldr w1, [x28, #64] 0.00 : ffff800010125f7c: orr w1, w1, #0x10 0.00 : ffff800010125f80: str w1, [x28, #64] 0.00 : ffff800010125f84: b ffff800010125834 : bitmap_subset(): : const unsigned long *src2, unsigned int nbits) : { : if (small_const_nbits(nbits)) : return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits)); : else : return __bitmap_subset(src1, src2, nbits); 0.00 : ffff800010125f88: adrp x19, ffff800011afb000 0.00 : ffff800010125f8c: add x19, x19, #0xfc0 0.00 : ffff800010125f90: add x1, x0, #0x88 0.00 : ffff800010125f94: mov w2, #0x100 // #256 0.00 : ffff800010125f98: mov x0, x19 0.00 : ffff800010125f9c: bl ffff8000104852f0 <__bitmap_subset> : update_sd_lb_stats(): : if ((env->flags & LBF_NOHZ_AGAIN) && 0.00 : ffff800010125fa0: cbz w0, ffff800010125fb4 : WRITE_ONCE(nohz.next_blocked, 0.00 : ffff800010125fa4: adrp x0, ffff800011907000 0.00 : ffff800010125fa8: ldr x0, [x0, #2432] 0.00 : ffff800010125fac: add x0, x0, #0x8 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010125fb0: str x0, [x19, #48] 0.00 : ffff800010125fb4: ldr x10, [x28] 0.00 : ffff800010125fb8: b ffff800010125b3c : calculate_imbalance(): : env->migration_type = migrate_misfit; 0.00 : ffff800010125fbc: mov w1, #0x3 // #3 : env->imbalance = 1; 0.00 : ffff800010125fc0: mov x2, #0x1 // #1 0.00 : ffff800010125fc4: str x2, [x28, #48] : env->migration_type = migrate_misfit; 0.00 : ffff800010125fc8: str w1, [x28, #84] : find_busiest_group(): : } 0.23 : ffff800010125fcc: adrp x1, ffff800011909000 0.00 : ffff800010125fd0: add x27, x1, #0x908 0.00 : ffff800010125fd4: ldr x2, [x29, #392] 0.00 : ffff800010125fd8: ldr x1, [x27] 0.00 : ffff800010125fdc: eor x1, x2, x1 0.00 : ffff800010125fe0: cbnz x1, ffff8000101261a4 0.00 : ffff800010125fe4: ldp x19, x20, [sp, #16] 0.00 : ffff800010125fe8: ldp x21, x22, [sp, #32] 0.00 : ffff800010125fec: ldp x23, x24, [sp, #48] 0.00 : ffff800010125ff0: ldp x25, x26, [sp, #64] 0.00 : ffff800010125ff4: ldp x27, x28, [sp, #80] 0.00 : ffff800010125ff8: ldp x29, x30, [sp], #400 0.00 : ffff800010125ffc: ret : update_sd_lb_stats(): : struct root_domain *rd = env->dst_rq->rd; 0.00 : ffff800010126000: ldr x1, [x28, #24] : WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD); 0.00 : ffff800010126004: and w4, w27, #0x1 : struct root_domain *rd = env->dst_rq->rd; 0.00 : ffff800010126008: ldr x1, [x1, #2464] : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001012600c: str w4, [x1, #88] 0.00 : ffff800010126010: str w0, [x1, #92] 0.00 : ffff800010126014: b ffff800010125b7c : update_sd_pick_busiest(): : if (sgs->group_type < busiest->group_type) 0.00 : ffff800010126018: cmp w2, #0x5 0.00 : ffff80001012601c: b.ne ffff800010125cf4 // b.any : if (sgs->avg_load <= busiest->avg_load) 0.00 : ffff800010126020: ldr x2, [x29, #248] 0.00 : ffff800010126024: mov x10, x0 0.00 : ffff800010126028: cmp x1, x2 0.00 : ffff80001012602c: b.hi ffff800010125c8c // b.pmore 0.00 : ffff800010126030: b ffff800010125af4 : calculate_imbalance(): : if (busiest->group_type > group_fully_busy) { 0.00 : ffff800010126034: cmp w1, #0x1 0.00 : ffff800010126038: b.hi ffff800010126090 // b.pmore : if (busiest->group_weight == 1 || sds->prefer_sibling) { 0.00 : ffff80001012603c: ldr w0, [x29, #292] 0.00 : ffff800010126040: cmp w0, #0x1 0.00 : ffff800010126044: b.eq ffff80001012616c // b.none 0.00 : ffff800010126048: ldr w0, [x29, #240] 0.00 : ffff80001012604c: cbnz w0, ffff80001012616c : env->imbalance = max_t(long, 0, (local->idle_cpus - 0.00 : ffff800010126050: ldr w1, [x29, #288] : env->migration_type = migrate_task; 0.00 : ffff800010126054: mov w2, #0x2 // #2 : env->imbalance = max_t(long, 0, (local->idle_cpus - 0.00 : ffff800010126058: ldr w0, [x29, #360] : env->migration_type = migrate_task; 0.00 : ffff80001012605c: str w2, [x28, #84] : env->imbalance = max_t(long, 0, (local->idle_cpus - 0.00 : ffff800010126060: sub w0, w0, w1 0.00 : ffff800010126064: lsr w0, w0, #1 0.00 : ffff800010126068: str x0, [x28, #48] : if (env->sd->flags & SD_NUMA) { 0.00 : ffff80001012606c: ldr x0, [x28] 0.00 : ffff800010126070: ldr w0, [x0, #56] 0.00 : ffff800010126074: tbz w0, #14, ffff800010126084 : if (busiest->sum_nr_running <= imbalance_min) 0.00 : ffff800010126078: ldr w0, [x29, #280] 0.00 : ffff80001012607c: cmp w0, #0x2 0.00 : ffff800010126080: b.ls ffff800010125e1c // b.plast 0.00 : ffff800010126084: ldr x0, [x28, #48] 0.00 : ffff800010126088: b ffff800010125bfc : if (local->group_type == group_has_spare) { 0.00 : ffff80001012608c: cbnz w2, ffff800010125bbc : env->imbalance = max(local->group_capacity, local->group_util) - 0.00 : ffff800010126090: ldp x0, x1, [x29, #336] : env->migration_type = migrate_util; 0.00 : ffff800010126094: mov w4, #0x1 // #1 : if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) { 0.00 : ffff800010126098: ldr w2, [x28, #44] : env->imbalance = max(local->group_capacity, local->group_util) - 0.00 : ffff80001012609c: cmp x0, x1 : env->migration_type = migrate_util; 0.00 : ffff8000101260a0: str w4, [x28, #84] : env->imbalance = max(local->group_capacity, local->group_util) - 0.00 : ffff8000101260a4: csel x0, x0, x1, cs // cs = hs, nlast 0.00 : ffff8000101260a8: sub x0, x0, x1 0.00 : ffff8000101260ac: str x0, [x28, #48] : if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) { 0.00 : ffff8000101260b0: cmp x0, #0x0 0.00 : ffff8000101260b4: ccmp w2, w4, #0x4, eq // eq = none 0.00 : ffff8000101260b8: b.eq ffff800010125bfc // b.none : env->imbalance = 1; 0.00 : ffff8000101260bc: mov x0, #0x1 // #1 : env->migration_type = migrate_task; 0.00 : ffff8000101260c0: mov w1, #0x2 // #2 : env->imbalance = 1; 0.00 : ffff8000101260c4: str x0, [x28, #48] 0.00 : ffff8000101260c8: mov x0, x3 : env->migration_type = migrate_task; 0.00 : ffff8000101260cc: str w1, [x28, #84] 0.00 : ffff8000101260d0: b ffff800010125fcc : find_busiest_group(): : busiest->sum_nr_running > local->sum_nr_running + 1) 0.00 : ffff8000101260d4: ldr w4, [x29, #352] : if (sds.prefer_sibling && local->group_type == group_has_spare && 0.00 : ffff8000101260d8: ldr w5, [x29, #280] : busiest->sum_nr_running > local->sum_nr_running + 1) 0.00 : ffff8000101260dc: add w4, w4, #0x1 : if (sds.prefer_sibling && local->group_type == group_has_spare && 0.00 : ffff8000101260e0: cmp w5, w4 0.00 : ffff8000101260e4: b.hi ffff800010125ba4 // b.pmore 0.00 : ffff8000101260e8: b ffff800010125eb0 : calculate_imbalance(): : env->imbalance = busiest->sum_h_nr_running; 0.00 : ffff8000101260ec: ldr w0, [x29, #284] : env->migration_type = migrate_task; 0.00 : ffff8000101260f0: mov w1, #0x2 // #2 : env->imbalance = busiest->sum_h_nr_running; 0.00 : ffff8000101260f4: str x0, [x28, #48] : env->migration_type = migrate_task; 0.00 : ffff8000101260f8: str w1, [x28, #84] 0.00 : ffff8000101260fc: b ffff800010125bfc : find_busiest_group(): : if (local->avg_load >= busiest->avg_load) 0.00 : ffff800010126100: ldr x6, [x29, #248] 0.00 : ffff800010126104: ldr x5, [x29, #320] 0.00 : ffff800010126108: cmp x5, x6 0.00 : ffff80001012610c: b.cs ffff800010125e1c // b.hs, b.nlast : sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010126110: ldp x4, x8, [x29, #216] 0.00 : ffff800010126114: lsl x4, x4, #10 0.00 : ffff800010126118: udiv x4, x4, x8 0.00 : ffff80001012611c: str x4, [x29, #232] : if (local->avg_load >= sds.avg_load) 0.00 : ffff800010126120: cmp x5, x4 0.00 : ffff800010126124: b.cs ffff800010125e1c // b.hs, b.nlast : env->sd->imbalance_pct * local->avg_load) 0.00 : ffff800010126128: ldr x8, [x28] : if (100 * busiest->avg_load <= 0.00 : ffff80001012612c: add x4, x6, x6, lsl #1 0.00 : ffff800010126130: add x6, x6, x4, lsl #3 : env->sd->imbalance_pct * local->avg_load) 0.00 : ffff800010126134: ldr w4, [x8, #44] 0.00 : ffff800010126138: mul x5, x4, x5 : if (100 * busiest->avg_load <= 0.00 : ffff80001012613c: cmp x5, x6, lsl #2 0.00 : ffff800010126140: b.cc ffff800010125eb0 // b.lo, b.ul, b.last : out_balanced: 0.00 : ffff800010126144: b ffff800010125e1c : calculate_imbalance(): : sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010126148: ldp x0, x2, [x29, #216] : local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff80001012614c: ldr x1, [x29, #328] : sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010126150: lsl x0, x0, #10 : local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010126154: lsl x1, x1, #10 : sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010126158: udiv x2, x0, x2 : local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff80001012615c: udiv x1, x1, x4 : sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010126160: str x2, [x29, #232] : local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 0.00 : ffff800010126164: str x1, [x29, #320] 0.00 : ffff800010126168: b ffff800010125bd0 : unsigned int nr_diff = busiest->sum_nr_running; 0.00 : ffff80001012616c: ldr w0, [x29, #280] : env->migration_type = migrate_task; 0.00 : ffff800010126170: mov w2, #0x2 // #2 : lsub_positive(&nr_diff, local->sum_nr_running); 0.00 : ffff800010126174: ldr w1, [x29, #352] : env->migration_type = migrate_task; 0.00 : ffff800010126178: str w2, [x28, #84] : lsub_positive(&nr_diff, local->sum_nr_running); 0.00 : ffff80001012617c: cmp w1, w0 0.00 : ffff800010126180: csel w1, w1, w0, ls // ls = plast 0.00 : ffff800010126184: sub w0, w0, w1 : env->imbalance = nr_diff >> 1; 0.00 : ffff800010126188: lsr w0, w0, #1 0.00 : ffff80001012618c: str x0, [x28, #48] 0.00 : ffff800010126190: b ffff80001012606c : update_sd_pick_busiest(): : else if ((sgs->idle_cpus == busiest->idle_cpus) && 0.00 : ffff800010126194: ldr w1, [x29, #280] 0.00 : ffff800010126198: cmp w6, w1 0.00 : ffff80001012619c: b.hi ffff800010125e50 // b.pmore 0.00 : ffff8000101261a0: b ffff800010125af0 : find_busiest_group(): : } 0.00 : ffff8000101261a4: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (403 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd8460 <_raw_spin_lock_irq>: : _raw_spin_lock_irq(): : EXPORT_SYMBOL(_raw_spin_lock_irqsave); : #endif : : #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ : void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) : { 0.00 : ffff800010cd8460: stp x29, x30, [sp, #-16]! 0.00 : ffff800010cd8464: mov x3, x0 : arch_local_irq_disable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff800010cd8468: mov x0, #0x60 // #96 : _raw_spin_lock_irq(): 0.00 : ffff800010cd846c: mov x29, sp : arch_local_irq_disable(): 1.30 : ffff800010cd8470: msr daifset, #0x2 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 10.63 : ffff800010cd8474: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.77 : ffff800010cd8478: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010cd847c: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 6.23 : ffff800010cd8480: str w0, [x1, #16] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010cd8484: b ffff800010cd84b0 <_raw_spin_lock_irq+0x50> 0.25 : ffff800010cd8488: b ffff800010cd84b0 <_raw_spin_lock_irq+0x50> : __lse__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, ) : __CMPXCHG_CASE(w, , , 32, ) : __CMPXCHG_CASE(x, , , 64, ) : __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") : __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 0.00 : ffff800010cd848c: mov w1, #0x0 // #0 0.00 : ffff800010cd8490: mov x0, x3 0.00 : ffff800010cd8494: mov w2, #0x1 // #1 0.00 : ffff800010cd8498: mov w4, w1 0.00 : ffff800010cd849c: casa w4, w2, [x3] 80.81 : ffff800010cd84a0: mov w0, w4 : atomic_try_cmpxchg_acquire(): : static inline bool : atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) : { : int r, o = *old; : r = atomic_cmpxchg_acquire(v, o, new); : if (unlikely(r != o)) 0.00 : ffff800010cd84a4: cbnz w0, ffff800010cd84c0 <_raw_spin_lock_irq+0x60> : _raw_spin_lock_irq(): : __raw_spin_lock_irq(lock); : } 0.00 : ffff800010cd84a8: ldp x29, x30, [sp], #16 0.00 : ffff800010cd84ac: ret : __ll_sc__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, , , , , K) : __CMPXCHG_CASE(w, , , 32, , , , , K) : __CMPXCHG_CASE( , , , 64, , , , , L) : __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) : __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 0.00 : ffff800010cd84b0: mov x1, #0x0 // #0 0.00 : ffff800010cd84b4: mov w2, #0x1 // #1 0.00 : ffff800010cd84b8: b ffff800010cd88f8 <_raw_read_lock_irqsave+0x220> : atomic_try_cmpxchg_acquire(): 0.00 : ffff800010cd84bc: cbz w0, ffff800010cd84a8 <_raw_spin_lock_irq+0x48> : queued_spin_lock(): : u32 val = 0; : : if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL))) : return; : : queued_spin_lock_slowpath(lock, val); 0.00 : ffff800010cd84c0: mov w1, w0 0.00 : ffff800010cd84c4: mov x0, x3 0.00 : ffff800010cd84c8: bl ffff800010139e90 : _raw_spin_lock_irq(): 0.00 : ffff800010cd84cc: ldp x29, x30, [sp], #16 0.00 : ffff800010cd84d0: ret Percent | Source code & Disassembly of vmlinux for cycles (363 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016ea30 : : ktime_get_update_offsets_now(): : * : * Called from hrtimer_interrupt() or retrigger_next_event() : */ : ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real, : ktime_t *offs_boot, ktime_t *offs_tai) : { 0.00 : ffff80001016ea30: stp x29, x30, [sp, #-96]! 0.00 : ffff80001016ea34: mov x29, sp 0.00 : ffff80001016ea38: stp x19, x20, [sp, #16] 0.00 : ffff80001016ea3c: mov x20, x1 0.00 : ffff80001016ea40: stp x21, x22, [sp, #32] : *offs_tai = tk->offs_tai; : } : : /* Handle leapsecond insertion adjustments */ : if (unlikely(base >= tk->next_leap_ktime)) : *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); 0.00 : ffff80001016ea44: mov x22, #0xffffffffffff3600 // #-51712 : { 0.00 : ffff80001016ea48: stp x23, x24, [sp, #48] 0.00 : ffff80001016ea4c: mov x21, x3 0.00 : ffff80001016ea50: stp x25, x26, [sp, #64] 0.00 : ffff80001016ea54: adrp x24, ffff800011b20000 <__log_buf+0x1fc10> 0.00 : ffff80001016ea58: mov x25, x0 0.00 : ffff80001016ea5c: mov x23, x2 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001016ea60: add x24, x24, #0xec0 : ktime_get_update_offsets_now(): : *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); 0.00 : ffff80001016ea64: movk x22, #0xc465, lsl #16 : { 0.00 : ffff80001016ea68: str x27, [sp, #80] : __read_once_size(): 1.08 : ffff80001016ea6c: ldr w27, [x24, #64] : __read_seqcount_begin(): : { : unsigned ret; : : repeat: : ret = READ_ONCE(s->sequence); : if (unlikely(ret & 1)) { 0.00 : ffff80001016ea70: tbnz w27, #0, ffff80001016eb10 : raw_read_seqcount_begin(): : * section is tested by checking read_seqcount_retry function. : */ : static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) : { : unsigned ret = __read_seqcount_begin(s); : smp_rmb(); 29.66 : ffff80001016ea74: dmb ishld : __read_once_size(): 2.24 : ffff80001016ea78: ldr x1, [x24, #72] : ktime_get_update_offsets_now(): : base = tk->tkr_mono.base; 0.27 : ffff80001016ea7c: ldr x26, [x24, #112] : tk_clock_read(): : return clock->read(clock); 0.00 : ffff80001016ea80: mov x0, x1 0.00 : ffff80001016ea84: ldr x1, [x1] 0.00 : ffff80001016ea88: blr x1 : clocksource_delta(): : return ret & ~(mask >> 1) ? 0 : ret; : } : #else : static inline u64 clocksource_delta(u64 now, u64 last, u64 mask) : { : return (now - last) & mask; 0.00 : ffff80001016ea8c: ldp x1, x4, [x24, #80] : timekeeping_delta_to_ns(): : nsec >>= tkr->shift; 0.29 : ffff80001016ea90: ldp w6, w2, [x24, #96] : clocksource_delta(): 0.00 : ffff80001016ea94: sub x0, x0, x4 : timekeeping_delta_to_ns(): : nsec = delta * tkr->mult + tkr->xtime_nsec; 0.00 : ffff80001016ea98: ldr x5, [x24, #104] : clocksource_delta(): 0.00 : ffff80001016ea9c: and x0, x0, x1 : ktime_get_update_offsets_now(): : if (*cwsseq != tk->clock_was_set_seq) { 0.00 : ffff80001016eaa0: ldr w3, [x25] 0.57 : ffff80001016eaa4: ldr w1, [x24, #244] : timekeeping_delta_to_ns(): : nsec = delta * tkr->mult + tkr->xtime_nsec; 0.00 : ffff80001016eaa8: madd x0, x0, x6, x5 : ktime_get_update_offsets_now(): : if (*cwsseq != tk->clock_was_set_seq) { 0.00 : ffff80001016eaac: cmp w3, w1 : timekeeping_delta_to_ns(): : nsec >>= tkr->shift; 0.00 : ffff80001016eab0: lsr x0, x0, x2 : ktime_get_update_offsets_now(): : base = ktime_add_ns(base, nsecs); 4.98 : ffff80001016eab4: add x0, x0, x26 : if (*cwsseq != tk->clock_was_set_seq) { 0.00 : ffff80001016eab8: b.eq ffff80001016ead8 // b.none : *cwsseq = tk->clock_was_set_seq; 0.00 : ffff80001016eabc: str w1, [x25] : *offs_real = tk->offs_real; 0.00 : ffff80001016eac0: ldr x1, [x24, #216] 0.00 : ffff80001016eac4: str x1, [x20] : *offs_boot = tk->offs_boot; 0.00 : ffff80001016eac8: ldr x1, [x24, #224] 0.00 : ffff80001016eacc: str x1, [x23] : *offs_tai = tk->offs_tai; 0.00 : ffff80001016ead0: ldr x1, [x24, #232] 0.00 : ffff80001016ead4: str x1, [x21] : if (unlikely(base >= tk->next_leap_ktime)) 0.00 : ffff80001016ead8: ldr x2, [x24, #256] 0.00 : ffff80001016eadc: cmp x2, x0 0.00 : ffff80001016eae0: b.le ffff80001016eb18 : read_seqcount_retry(): : * If the critical section was invalid, it must be ignored (and typically : * retried). : */ : static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) : { : smp_rmb(); 1.69 : ffff80001016eae4: dmb ishld : ktime_get_update_offsets_now(): : : } while (read_seqcount_retry(&tk_core.seq, seq)); 1.95 : ffff80001016eae8: ldr w1, [x24, #64] 0.00 : ffff80001016eaec: cmp w27, w1 0.00 : ffff80001016eaf0: b.ne ffff80001016ea6c // b.any : : return base; : } 0.00 : ffff80001016eaf4: ldp x19, x20, [sp, #16] 0.00 : ffff80001016eaf8: ldp x21, x22, [sp, #32] 0.00 : ffff80001016eafc: ldp x23, x24, [sp, #48] 0.27 : ffff80001016eb00: ldp x25, x26, [sp, #64] 0.00 : ffff80001016eb04: ldr x27, [sp, #80] 0.00 : ffff80001016eb08: ldp x29, x30, [sp], #96 0.00 : ffff80001016eb0c: ret : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 38.83 : ffff80001016eb10: yield 18.17 : ffff80001016eb14: b ffff80001016ea6c : ktime_get_update_offsets_now(): : *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); 0.00 : ffff80001016eb18: ldr x1, [x24, #216] 0.00 : ffff80001016eb1c: add x1, x1, x22 0.00 : ffff80001016eb20: str x1, [x20] 0.00 : ffff80001016eb24: b ffff80001016eae4 Percent | Source code & Disassembly of vmlinux for cycles (334 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016b278 <__hrtimer_run_queues>: : __hrtimer_run_queues(): : : static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, : unsigned long flags, unsigned int active_mask) : { : struct hrtimer_clock_base *base; : unsigned int active = cpu_base->active_bases & active_mask; 0.00 : ffff80001016b278: ldr w4, [x0, #8] : __next_base(): : if (!*active) 0.00 : ffff80001016b27c: ands w3, w3, w4 0.00 : ffff80001016b280: b.eq ffff80001016b428 <__hrtimer_run_queues+0x1b0> // b.none : __hrtimer_run_queues(): : { 0.31 : ffff80001016b284: stp x29, x30, [sp, #-112]! 0.00 : ffff80001016b288: mov x29, sp 0.00 : ffff80001016b28c: str x22, [sp, #40] : __next_base(): : idx = __ffs(*active); 0.00 : ffff80001016b290: mov w22, w3 : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 0.00 : ffff80001016b294: rbit x22, x22 : __hrtimer_run_queues(): : { 0.00 : ffff80001016b298: str x19, [sp, #16] : __ffs(): 0.00 : ffff80001016b29c: clz x22, x22 : __hrtimer_run_queues(): 0.00 : ffff80001016b2a0: stp x25, x26, [sp, #64] : __next_base(): : return &cpu_base->clock_base[idx]; 0.00 : ffff80001016b2a4: add x19, x22, #0x1 : *active &= ~(1U << idx); 0.00 : ffff80001016b2a8: mov w26, #0x1 // #1 0.00 : ffff80001016b2ac: lsl w25, w26, w22 0.00 : ffff80001016b2b0: bic w25, w3, w25 : __hrtimer_run_queues(): : : for_each_active_base(base, cpu_base, active) { 0.00 : ffff80001016b2b4: adds x19, x0, x19, lsl #6 0.00 : ffff80001016b2b8: b.eq ffff80001016b40c <__hrtimer_run_queues+0x194> // b.none 0.00 : ffff80001016b2bc: stp x20, x21, [x29, #24] 0.00 : ffff80001016b2c0: mov x21, x0 0.61 : ffff80001016b2c4: stp x23, x24, [x29, #48] 0.00 : ffff80001016b2c8: mov x24, x2 0.00 : ffff80001016b2cc: stp x27, x28, [x29, #80] 0.00 : ffff80001016b2d0: str x1, [x29, #104] 0.00 : ffff80001016b2d4: nop 0.00 : ffff80001016b2d8: mov w22, w22 : struct timerqueue_node *node; : ktime_t basenow; : : basenow = ktime_add(now, base->offset); : : while ((node = timerqueue_getnext(&base->active))) { 0.30 : ffff80001016b2dc: ldr x20, [x19, #40] : basenow = ktime_add(now, base->offset); 0.30 : ffff80001016b2e0: ldr x0, [x29, #104] 0.00 : ffff80001016b2e4: add x28, x21, x22, lsl #6 0.61 : ffff80001016b2e8: ldr x23, [x28, #120] 0.00 : ffff80001016b2ec: add x23, x0, x23 : while ((node = timerqueue_getnext(&base->active))) { 0.00 : ffff80001016b2f0: cbz x20, ffff80001016b3dc <__hrtimer_run_queues+0x164> : * BST we already have. : * We don't add extra wakeups by delaying timers that : * are right-of a not yet expired timer, because that : * timer will have to trigger a wakeup anyway. : */ : if (basenow < hrtimer_get_softexpires_tv64(timer)) 0.00 : ffff80001016b2f4: ldr x0, [x20, #32] 0.00 : ffff80001016b2f8: cmp x23, x0 0.00 : ffff80001016b2fc: b.lt ffff80001016b3dc <__hrtimer_run_queues+0x164> // b.tstop : enqueue_hrtimer(): : return timerqueue_add(&base->active, &timer->node); 0.60 : ffff80001016b300: add x22, x21, x22, lsl #6 0.00 : ffff80001016b304: add x22, x22, #0x60 0.00 : ffff80001016b308: b ffff80001016b34c <__hrtimer_run_queues+0xd4> : raw_write_seqcount_barrier(): : * X = false; : * } : */ : static inline void raw_write_seqcount_barrier(seqcount_t *s) : { : s->sequence++; 0.00 : ffff80001016b30c: ldr w0, [x19, #16] 0.00 : ffff80001016b310: add w0, w0, #0x1 0.30 : ffff80001016b314: str w0, [x19, #16] : smp_wmb(); 0.00 : ffff80001016b318: dmb ishst : s->sequence++; 43.24 : ffff80001016b31c: ldr w3, [x19, #16] 0.00 : ffff80001016b320: add w3, w3, #0x1 0.00 : ffff80001016b324: str w3, [x19, #16] : __run_hrtimer(): : WARN_ON_ONCE(base->running != timer); 0.00 : ffff80001016b328: ldr x0, [x28, #88] 0.00 : ffff80001016b32c: cmp x0, x20 0.00 : ffff80001016b330: b.ne ffff80001016b420 <__hrtimer_run_queues+0x1a8> // b.any : base->running = NULL; 0.00 : ffff80001016b334: str xzr, [x28, #88] : __hrtimer_run_queues(): : while ((node = timerqueue_getnext(&base->active))) { 0.30 : ffff80001016b338: ldr x20, [x19, #40] 0.00 : ffff80001016b33c: cbz x20, ffff80001016b3dc <__hrtimer_run_queues+0x164> : if (basenow < hrtimer_get_softexpires_tv64(timer)) 0.00 : ffff80001016b340: ldr x0, [x20, #32] 0.00 : ffff80001016b344: cmp x23, x0 0.00 : ffff80001016b348: b.lt ffff80001016b3dc <__hrtimer_run_queues+0x164> // b.tstop : __run_hrtimer(): : base->running = timer; 0.00 : ffff80001016b34c: str x20, [x28, #88] : raw_write_seqcount_barrier(): : s->sequence++; 0.00 : ffff80001016b350: ldr w0, [x19, #16] 0.00 : ffff80001016b354: add w0, w0, #0x1 0.00 : ffff80001016b358: str w0, [x19, #16] : smp_wmb(); 0.00 : ffff80001016b35c: dmb ishst : s->sequence++; 48.41 : ffff80001016b360: ldr w0, [x19, #16] : __run_hrtimer(): : __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0); 0.00 : ffff80001016b364: mov w3, #0x0 // #0 0.00 : ffff80001016b368: mov w2, #0x0 // #0 0.00 : ffff80001016b36c: mov x1, x19 : raw_write_seqcount_barrier(): 0.00 : ffff80001016b370: add w0, w0, #0x1 0.00 : ffff80001016b374: str w0, [x19, #16] : __run_hrtimer(): 0.00 : ffff80001016b378: mov x0, x20 0.00 : ffff80001016b37c: bl ffff80001016b130 <__remove_hrtimer> : fn = timer->function; 0.00 : ffff80001016b380: ldr x27, [x20, #40] : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 0.00 : ffff80001016b384: mov x1, x24 0.00 : ffff80001016b388: mov x0, x21 0.00 : ffff80001016b38c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : restart = fn(timer); 0.00 : ffff80001016b390: mov x0, x20 0.00 : ffff80001016b394: blr x27 0.00 : ffff80001016b398: mov w27, w0 : raw_spin_lock_irq(&cpu_base->lock); 0.00 : ffff80001016b39c: mov x0, x21 0.00 : ffff80001016b3a0: bl ffff800010cd8460 <_raw_spin_lock_irq> : if (restart != HRTIMER_NORESTART && 0.28 : ffff80001016b3a4: cbz w27, ffff80001016b30c <__hrtimer_run_queues+0x94> 0.00 : ffff80001016b3a8: ldrb w0, [x20, #56] 0.00 : ffff80001016b3ac: tbnz w0, #0, ffff80001016b30c <__hrtimer_run_queues+0x94> : enqueue_hrtimer(): : base->cpu_base->active_bases |= 1 << base->index; 0.00 : ffff80001016b3b0: ldr x4, [x28, #64] : return timerqueue_add(&base->active, &timer->node); 0.00 : ffff80001016b3b4: mov x1, x20 : base->cpu_base->active_bases |= 1 << base->index; 0.31 : ffff80001016b3b8: ldr w3, [x28, #72] : return timerqueue_add(&base->active, &timer->node); 0.00 : ffff80001016b3bc: mov x0, x22 : base->cpu_base->active_bases |= 1 << base->index; 0.60 : ffff80001016b3c0: ldr w2, [x4, #8] 0.00 : ffff80001016b3c4: lsl w3, w26, w3 0.00 : ffff80001016b3c8: orr w2, w2, w3 2.62 : ffff80001016b3cc: str w2, [x4, #8] : __write_once_size(): : } : : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff80001016b3d0: strb w26, [x20, #56] : enqueue_hrtimer(): : return timerqueue_add(&base->active, &timer->node); 0.00 : ffff80001016b3d4: bl ffff800010cc8aa0 0.00 : ffff80001016b3d8: b ffff80001016b30c <__hrtimer_run_queues+0x94> : __next_base(): : if (!*active) 0.30 : ffff80001016b3dc: cbz w25, ffff80001016b400 <__hrtimer_run_queues+0x188> : idx = __ffs(*active); 0.00 : ffff80001016b3e0: mov w22, w25 : __ffs(): 0.00 : ffff80001016b3e4: rbit x22, x22 0.00 : ffff80001016b3e8: clz x22, x22 : __next_base(): : return &cpu_base->clock_base[idx]; 0.00 : ffff80001016b3ec: add x19, x22, #0x1 : *active &= ~(1U << idx); 0.00 : ffff80001016b3f0: lsl w0, w26, w22 0.00 : ffff80001016b3f4: bic w25, w25, w0 : __hrtimer_run_queues(): : for_each_active_base(base, cpu_base, active) { 0.00 : ffff80001016b3f8: adds x19, x21, x19, lsl #6 0.00 : ffff80001016b3fc: b.ne ffff80001016b2d8 <__hrtimer_run_queues+0x60> // b.any 0.00 : ffff80001016b400: ldp x20, x21, [x29, #24] 0.61 : ffff80001016b404: ldp x23, x24, [x29, #48] 0.00 : ffff80001016b408: ldp x27, x28, [x29, #80] : __run_hrtimer(cpu_base, base, timer, &basenow, flags); : if (active_mask == HRTIMER_ACTIVE_SOFT) : hrtimer_sync_wait_running(cpu_base, flags); : } : } : } 0.00 : ffff80001016b40c: ldr x19, [sp, #16] 0.00 : ffff80001016b410: ldr x22, [sp, #40] 0.30 : ffff80001016b414: ldp x25, x26, [sp, #64] 0.00 : ffff80001016b418: ldp x29, x30, [sp], #112 0.00 : ffff80001016b41c: ret : __run_hrtimer(): : WARN_ON_ONCE(base->running != timer); 0.00 : ffff80001016b420: brk #0x800 0.00 : ffff80001016b424: b ffff80001016b334 <__hrtimer_run_queues+0xbc> 0.00 : ffff80001016b428: ret Percent | Source code & Disassembly of vmlinux for cycles (347 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011ae08 : : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011ae08: adrp x1, ffff800011afb000 0.00 : ffff80001011ae0c: add x1, x1, #0xf48 : calc_load_nohz_stop(): : : /* : * If we're still before the pending sample window, we're done. : */ : this_rq->calc_load_update = READ_ONCE(calc_load_update); : if (time_before(jiffies, this_rq->calc_load_update)) 0.00 : ffff80001011ae10: adrp x3, ffff800011907000 : struct rq *this_rq = this_rq(); 0.00 : ffff80001011ae14: adrp x0, ffff800011528000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011ae18: mrs x2, tpidr_el1 : calc_load_nohz_stop(): 0.00 : ffff80001011ae1c: add x0, x0, #0xe80 : __read_once_size(): 2.88 : ffff80001011ae20: ldr x1, [x1, #8] : calc_load_nohz_stop(): 0.00 : ffff80001011ae24: add x0, x0, x2 : if (time_before(jiffies, this_rq->calc_load_update)) 97.12 : ffff80001011ae28: ldr x2, [x3, #2432] 0.00 : ffff80001011ae2c: sub x2, x2, x1 : this_rq->calc_load_update = READ_ONCE(calc_load_update); 0.00 : ffff80001011ae30: str x1, [x0, #2856] : if (time_before(jiffies, this_rq->calc_load_update)) 0.00 : ffff80001011ae34: tbnz x2, #63, ffff80001011ae48 : /* : * We woke inside or after the sample window, this means we're already : * accounted through the nohz accounting, so skip the entire deal and : * sync up for the next window. : */ : if (time_before(jiffies, this_rq->calc_load_update + 10)) 0.00 : ffff80001011ae38: ldr x2, [x3, #2432] 0.00 : ffff80001011ae3c: sub x2, x2, x1 0.00 : ffff80001011ae40: cmp x2, #0xa 0.00 : ffff80001011ae44: b.mi ffff80001011ae4c // b.first : this_rq->calc_load_update += LOAD_FREQ; : } 0.00 : ffff80001011ae48: ret : this_rq->calc_load_update += LOAD_FREQ; 0.00 : ffff80001011ae4c: add x1, x1, #0x4e3 0.00 : ffff80001011ae50: str x1, [x0, #2856] : } 0.00 : ffff80001011ae54: ret Percent | Source code & Disassembly of vmlinux for cycles (322 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101d75e8 : : mempool_free_slab(): : return kmem_cache_alloc(mem, gfp_mask); : } : EXPORT_SYMBOL(mempool_alloc_slab); : : void mempool_free_slab(void *element, void *pool_data) : { 0.00 : ffff8000101d75e8: stp x29, x30, [sp, #-16]! 0.00 : ffff8000101d75ec: mov x2, x1 : struct kmem_cache *mem = pool_data; : kmem_cache_free(mem, element); 0.00 : ffff8000101d75f0: mov x1, x0 0.00 : ffff8000101d75f4: mov x0, x2 : { 92.55 : ffff8000101d75f8: mov x29, sp : kmem_cache_free(mem, element); 0.00 : ffff8000101d75fc: bl ffff800010253518 : } 7.45 : ffff8000101d7600: ldp x29, x30, [sp], #16 0.00 : ffff8000101d7604: ret Percent | Source code & Disassembly of vmlinux for cycles (340 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012a428 : : balance_rt(): : requeue_task_rt(rq, p, 1); : resched_curr(rq); : } : : static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf) : { 1.17 : ffff80001012a428: stp x29, x30, [sp, #-32]! 0.00 : ffff80001012a42c: mov x29, sp 0.32 : ffff80001012a430: str x19, [sp, #16] 0.00 : ffff80001012a434: mov x19, x0 : if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) { 0.31 : ffff80001012a438: ldrh w2, [x1, #484] 0.00 : ffff80001012a43c: cbnz w2, ffff80001012a450 0.00 : ffff80001012a440: ldr w2, [x1, #108] 4.99 : ffff80001012a444: ldr w1, [x0, #2136] 0.00 : ffff80001012a448: cmp w2, w1 0.00 : ffff80001012a44c: b.lt ffff80001012a488 // b.tstop : rq_unpin_lock(rq, rf); : pull_rt_task(rq); : rq_repin_lock(rq, rf); : } : : return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq); 1.79 : ffff80001012a450: ldr x0, [x19, #2368] : sched_stop_runnable(): : extern const struct sched_class fair_sched_class; : extern const struct sched_class idle_sched_class; : : static inline bool sched_stop_runnable(struct rq *rq) : { : return rq->stop && task_on_rq_queued(rq->stop); 0.00 : ffff80001012a454: cbz x0, ffff80001012a464 : task_on_rq_queued(): : return p->on_rq == TASK_ON_RQ_QUEUED; 4.77 : ffff80001012a458: ldr w0, [x0, #104] : sched_stop_runnable(): : return rq->stop && task_on_rq_queued(rq->stop); 0.00 : ffff80001012a45c: cmp w0, #0x1 0.00 : ffff80001012a460: b.eq ffff80001012a47c // b.none : balance_rt(): 80.40 : ffff80001012a464: ldr x1, [x19, #2232] 1.53 : ffff80001012a468: mov w0, #0x1 // #1 0.00 : ffff80001012a46c: cbnz x1, ffff80001012a47c 0.88 : ffff80001012a470: ldr w0, [x19, #2184] 0.00 : ffff80001012a474: cmp w0, #0x0 0.00 : ffff80001012a478: cset w0, gt : } 0.27 : ffff80001012a47c: ldr x19, [sp, #16] 1.20 : ffff80001012a480: ldp x29, x30, [sp], #32 0.00 : ffff80001012a484: ret : pull_rt_task(rq); 0.00 : ffff80001012a488: bl ffff80001012a2d0 2.37 : ffff80001012a48c: b ffff80001012a450 Percent | Source code & Disassembly of vmlinux for cycles (333 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010117318 : : sched_ttwu_pending(): : return ret; : } : : #ifdef CONFIG_SMP : void sched_ttwu_pending(void) : { 0.00 : ffff800010117318: stp x29, x30, [sp, #-48]! : __xchg_case_mb_64(): : __XCHG_CASE(w, , rel_, 32, , , , , l, "memory") : __XCHG_CASE( , , rel_, 64, , , , , l, "memory") : __XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory") : __XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory") : __XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory") : __XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory") 0.00 : ffff80001011731c: mov x0, #0x0 // #0 : sched_ttwu_pending(): 4.59 : ffff800010117320: mov x29, sp 0.30 : ffff800010117324: stp x19, x20, [sp, #16] : struct rq *rq = this_rq(); 0.00 : ffff800010117328: adrp x19, ffff800011528000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 1.23 : ffff80001011732c: mrs x1, tpidr_el1 : sched_ttwu_pending(): 0.00 : ffff800010117330: add x19, x19, #0xe80 0.00 : ffff800010117334: add x19, x19, x1 : __xchg_case_mb_64(): 0.00 : ffff800010117338: add x2, x19, #0xba0 0.00 : ffff80001011733c: prfm pstl1strm, [x2] 89.63 : ffff800010117340: ldxr x20, [x2] 0.00 : ffff800010117344: stlxr w1, x0, [x2] 0.00 : ffff800010117348: cbnz w1, ffff800010117340 0.00 : ffff80001011734c: dmb ish : sched_ttwu_pending(): : struct llist_node *llist = llist_del_all(&rq->wake_list); : struct task_struct *p, *t; : struct rq_flags rf; : : if (!llist) 3.00 : ffff800010117350: cbnz x20, ffff800010117360 : : llist_for_each_entry_safe(p, t, llist, wake_entry) : ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); : : rq_unlock_irqrestore(rq, &rf); : } 0.00 : ffff800010117354: ldp x19, x20, [sp, #16] 1.25 : ffff800010117358: ldp x29, x30, [sp], #48 0.00 : ffff80001011735c: ret 0.00 : ffff800010117360: str x21, [x29, #32] : rq_lock_irqsave(): : : static inline void : rq_lock_irqsave(struct rq *rq, struct rq_flags *rf) : __acquires(rq->lock) : { : raw_spin_lock_irqsave(&rq->lock, rf->flags); 0.00 : ffff800010117364: mov x0, x19 0.00 : ffff800010117368: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff80001011736c: mov x21, x0 : update_rq_clock(): : if (rq->clock_update_flags & RQCF_ACT_SKIP) 0.00 : ffff800010117370: ldr w0, [x19, #2392] 0.00 : ffff800010117374: tbz w0, #1, ffff8000101173c4 : sched_ttwu_pending(): : llist_for_each_entry_safe(p, t, llist, wake_entry) 0.00 : ffff800010117378: sub x1, x20, #0x38 0.00 : ffff80001011737c: cbz x20, ffff8000101173a8 : ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 0.00 : ffff800010117380: ldrb w2, [x1, #1060] 0.00 : ffff800010117384: mov x0, x19 : llist_for_each_entry_safe(p, t, llist, wake_entry) 0.00 : ffff800010117388: ldr x20, [x1, #56] : ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 0.00 : ffff80001011738c: lsr w2, w2, #1 : llist_for_each_entry_safe(p, t, llist, wake_entry) 0.00 : ffff800010117390: sub x20, x20, #0x38 : ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 0.00 : ffff800010117394: and w2, w2, #0x4 0.00 : ffff800010117398: bl ffff800010115f40 0.00 : ffff80001011739c: mov x1, x20 : llist_for_each_entry_safe(p, t, llist, wake_entry) 0.00 : ffff8000101173a0: cmn x20, #0x38 0.00 : ffff8000101173a4: b.ne ffff800010117380 // b.any : rq_unlock_irqrestore(): : static inline void : rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf) : __releases(rq->lock) : { : rq_unpin_lock(rq, rf); : raw_spin_unlock_irqrestore(&rq->lock, rf->flags); 0.00 : ffff8000101173a8: mov x1, x21 0.00 : ffff8000101173ac: mov x0, x19 0.00 : ffff8000101173b0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff8000101173b4: ldr x21, [x29, #32] : sched_ttwu_pending(): : } 0.00 : ffff8000101173b8: ldp x19, x20, [sp, #16] 0.00 : ffff8000101173bc: ldp x29, x30, [sp], #48 0.00 : ffff8000101173c0: ret : update_rq_clock(): 0.00 : ffff8000101173c4: mov x0, x19 0.00 : ffff8000101173c8: bl ffff800010113d48 0.00 : ffff8000101173cc: b ffff800010117378 Percent | Source code & Disassembly of vmlinux for cycles (614 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010252238 : : kmem_cache_alloc(): : { : return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr); : } : : void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) : { 5.17 : ffff800010252238: stp x29, x30, [sp, #-64]! : slab_pre_alloc_hook(): : } : : static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, : gfp_t flags) : { : flags &= gfp_allowed_mask; 0.00 : ffff80001025223c: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> : kmem_cache_alloc(): 0.17 : ffff800010252240: mov x29, sp 0.33 : ffff800010252244: stp x19, x20, [sp, #16] 0.00 : ffff800010252248: mov x20, x0 : slab_pre_alloc_hook(): 0.83 : ffff80001025224c: ldr w19, [x2, #1104] : kmem_cache_alloc(): 1.45 : ffff800010252250: stp x21, x22, [sp, #32] 0.00 : ffff800010252254: mov w21, w1 : slab_pre_alloc_hook(): 0.00 : ffff800010252258: and w19, w1, w19 0.00 : ffff80001025225c: mov x22, x30 : fs_reclaim_acquire(flags); : fs_reclaim_release(flags); : : might_sleep_if(gfpflags_allow_blocking(flags)); : : if (should_failslab(s, flags)) 0.33 : ffff800010252260: mov w1, w19 0.00 : ffff800010252264: bl ffff8000101feba0 0.47 : ffff800010252268: cbnz w0, ffff800010252450 0.66 : ffff80001025226c: stp x23, x24, [x29, #48] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010252270: nop : slab_alloc_node(): : if (!s) 10.20 : ffff800010252274: cbz x20, ffff800010252448 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.66 : ffff800010252278: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001025227c: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010252280: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.32 : ffff800010252284: str w0, [x1, #16] : slab_alloc_node(): : tid = this_cpu_read(s->cpu_slab->tid); 0.00 : ffff800010252288: ldr x0, [x20] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 3.14 : ffff80001025228c: mrs x2, tpidr_el1 : slab_alloc_node(): 0.00 : ffff800010252290: add x0, x0, #0x8 : __read_once_size(): : __READ_ONCE_SIZE; 0.16 : ffff800010252294: ldr x23, [x0, x2] 16.95 : ffff800010252298: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001025229c: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 2.12 : ffff8000102522a0: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102522a4: cbz x0, ffff800010252390 : __read_once_size(): : __READ_ONCE_SIZE; 0.31 : ffff8000102522a8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102522ac: cbz x0, ffff800010252390 : __my_cpu_offset(): 2.29 : ffff8000102522b0: mrs x1, tpidr_el1 : slab_alloc_node(): : c = raw_cpu_ptr(s->cpu_slab); 0.81 : ffff8000102522b4: ldr x0, [x20] 0.00 : ffff8000102522b8: add x2, x0, x1 : __read_once_size(): 0.00 : ffff8000102522bc: ldr x5, [x2, #8] : slab_alloc_node(): : unlikely(tid != READ_ONCE(c->tid))); 0.17 : ffff8000102522c0: cmp x5, x23 0.00 : ffff8000102522c4: b.ne ffff800010252278 // b.any : object = c->freelist; 1.15 : ffff8000102522c8: ldr x19, [x0, x1] : if (unlikely(!object || !node_match(page, node))) { 0.00 : ffff8000102522cc: cbz x19, ffff80001025242c : get_freepointer(): : return freelist_dereference(s, object + s->offset); 0.99 : ffff8000102522d0: ldr w2, [x20, #32] : get_current(): 0.83 : ffff8000102522d4: mrs x1, sp_el0 : __read_once_size(): 0.16 : ffff8000102522d8: ldr w0, [x1, #16] : freelist_dereference(): : return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr), 0.00 : ffff8000102522dc: ldr x24, [x19, x2] : __preempt_count_add(): : pc += val; 0.49 : ffff8000102522e0: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 19.98 : ffff8000102522e4: str w0, [x1, #16] : __my_cpu_offset(): 0.00 : ffff8000102522e8: mrs x0, tpidr_el1 : slab_alloc_node(): : if (unlikely(!this_cpu_cmpxchg_double( 0.00 : ffff8000102522ec: ldr x4, [x20] : next_tid(): : return tid + TID_STEP; 0.00 : ffff8000102522f0: add x3, x23, #0x100 : slab_alloc_node(): : if (unlikely(!this_cpu_cmpxchg_double( 0.00 : ffff8000102522f4: add x4, x4, x0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102522f8: b ffff8000102523a0 0.00 : ffff8000102522fc: b ffff8000102523a0 : __lse__cmpxchg_double(): : : cl); \ : \ : return x0; \ : } : : __CMPXCHG_DBL( , ) 0.48 : ffff800010252300: mov x0, x19 0.00 : ffff800010252304: mov x1, x5 0.00 : ffff800010252308: mov x2, x24 0.00 : ffff80001025230c: casp x0, x1, x2, x3, [x4] 20.45 : ffff800010252310: eor x0, x0, x19 0.00 : ffff800010252314: eor x1, x1, x5 0.00 : ffff800010252318: orr x0, x0, x1 0.00 : ffff80001025231c: mov x23, x0 : get_current(): 0.00 : ffff800010252320: mrs x0, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010252324: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010252328: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001025232c: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010252330: cbz x1, ffff800010252398 : __read_once_size(): : __READ_ONCE_SIZE; 0.96 : ffff800010252334: ldr x0, [x0, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010252338: cbz x0, ffff800010252398 : slab_alloc_node(): 0.16 : ffff80001025233c: cbnz x23, ffff800010252278 : prefetch_freepointer(): : prefetch(object + s->offset); 0.00 : ffff800010252340: ldr w0, [x20, #32] : prefetch(): : * Prefetching support : */ : #define ARCH_HAS_PREFETCH : static inline void prefetch(const void *ptr) : { : asm volatile("prfm pldl1keep, %a0\n" : : "p" (ptr)); 2.29 : ffff800010252344: prfm pldl1keep, [x24, x0] : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010252348: nop 1.29 : ffff80001025234c: nop : slab_want_init_on_alloc(): : return false; : if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) : return flags & __GFP_ZERO; : return true; : } : return flags & __GFP_ZERO; 0.00 : ffff800010252350: ubfx x21, x21, #8, #1 : slab_alloc_node(): : if (unlikely(slab_want_init_on_alloc(gfpflags, s)) && object) 0.00 : ffff800010252354: cbnz w21, ffff800010252414 : arch_static_branch(): 0.00 : ffff800010252358: nop 0.00 : ffff80001025235c: ldp x23, x24, [x29, #48] : kmem_cache_alloc(): : : trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, : s->size, gfpflags); : : return ret; : } 0.00 : ffff800010252360: mov x0, x19 0.00 : ffff800010252364: ldp x19, x20, [sp, #16] 0.00 : ffff800010252368: ldp x21, x22, [sp, #32] 0.00 : ffff80001025236c: ldp x29, x30, [sp], #64 0.00 : ffff800010252370: ret : slab_pre_alloc_hook(): : if (memcg_kmem_enabled() && 1.46 : ffff800010252374: tbnz w19, #22, ffff800010252380 : ((flags & __GFP_ACCOUNT) || (s->flags & SLAB_ACCOUNT))) 0.00 : ffff800010252378: ldr w0, [x20, #8] 0.00 : ffff80001025237c: tbz w0, #26, ffff800010252274 : return memcg_kmem_get_cache(s); 0.00 : ffff800010252380: mov x0, x20 0.00 : ffff800010252384: bl ffff80001026d6f8 0.00 : ffff800010252388: mov x20, x0 0.00 : ffff80001025238c: b ffff800010252274 : slab_alloc_node(): : tid = this_cpu_read(s->cpu_slab->tid); 0.00 : ffff800010252390: bl ffff800010cd2e78 0.00 : ffff800010252394: b ffff8000102522b0 : if (unlikely(!this_cpu_cmpxchg_double( 0.00 : ffff800010252398: bl ffff800010cd2e78 0.00 : ffff80001025239c: b ffff80001025233c : __ll_sc__cmpxchg_double(): : : cl); \ : \ : return ret; \ : } : : __CMPXCHG_DBL( , , , ) 0.00 : ffff8000102523a0: b ffff800010255b18 0.00 : ffff8000102523a4: b ffff800010252320 : slab_want_init_on_alloc(): : if (c->ctor) 0.00 : ffff8000102523a8: ldr x0, [x20, #64] 0.00 : ffff8000102523ac: cbnz x0, ffff800010252358 : if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) 0.00 : ffff8000102523b0: ldr w0, [x20, #8] 0.00 : ffff8000102523b4: and w0, w0, #0xff800 0.00 : ffff8000102523b8: and w0, w0, #0xfff80fff 0.00 : ffff8000102523bc: cbz w0, ffff800010252414 : return flags & __GFP_ZERO; 0.00 : ffff8000102523c0: ubfx x21, x21, #8, #1 : slab_alloc_node(): : if (unlikely(slab_want_init_on_alloc(gfpflags, s)) && object) 0.00 : ffff8000102523c4: cbz w21, ffff800010252358 0.00 : ffff8000102523c8: b ffff800010252414 : slab_post_alloc_hook(): : memcg_kmem_put_cache(s); 0.50 : ffff8000102523cc: mov x0, x20 0.00 : ffff8000102523d0: bl ffff80001026d8d8 : kmem_cache_alloc(): : } 0.00 : ffff8000102523d4: mov x0, x19 : return ret; 0.33 : ffff8000102523d8: ldp x23, x24, [x29, #48] : } 0.33 : ffff8000102523dc: ldp x19, x20, [sp, #16] 0.32 : ffff8000102523e0: ldp x21, x22, [sp, #32] 0.31 : ffff8000102523e4: ldp x29, x30, [sp], #64 0.00 : ffff8000102523e8: ret : slab_want_init_on_free(): : } : : static inline bool slab_want_init_on_free(struct kmem_cache *c) : { : if (static_branch_unlikely(&init_on_free)) : return !(c->ctor || 0.00 : ffff8000102523ec: ldr x0, [x20, #64] 0.00 : ffff8000102523f0: cbnz x0, ffff80001025234c : (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))); 0.00 : ffff8000102523f4: ldr w0, [x20, #8] 0.00 : ffff8000102523f8: and w0, w0, #0xff800 0.00 : ffff8000102523fc: and w0, w0, #0xfff80fff : return !(c->ctor || 0.00 : ffff800010252400: cbnz w0, ffff80001025234c : maybe_wipe_obj_freeptr(): : if (unlikely(slab_want_init_on_free(s)) && obj) 0.00 : ffff800010252404: cbz x19, ffff80001025234c : memset((void *)((char *)obj + s->offset), 0, sizeof(void *)); 0.00 : ffff800010252408: ldr w0, [x20, #32] 0.00 : ffff80001025240c: str xzr, [x19, x0] 0.00 : ffff800010252410: b ffff80001025234c : slab_alloc_node(): : if (unlikely(slab_want_init_on_alloc(gfpflags, s)) && object) 0.00 : ffff800010252414: cbz x19, ffff800010252358 : memset(object, 0, s->object_size); 0.33 : ffff800010252418: ldr w2, [x20, #28] 0.00 : ffff80001025241c: mov w1, #0x0 // #0 0.00 : ffff800010252420: mov x0, x19 0.00 : ffff800010252424: bl ffff800010cb80c0 <__memset> 0.30 : ffff800010252428: b ffff800010252358 : object = __slab_alloc(s, gfpflags, node, addr, c); 0.32 : ffff80001025242c: mov x3, x22 0.00 : ffff800010252430: mov w2, #0xffffffff // #-1 0.00 : ffff800010252434: mov w1, w21 0.00 : ffff800010252438: mov x0, x20 0.00 : ffff80001025243c: bl ffff800010251f98 <__slab_alloc.isra.97> 0.00 : ffff800010252440: mov x19, x0 0.00 : ffff800010252444: b ffff800010252348 0.00 : ffff800010252448: ldp x23, x24, [x29, #48] 0.00 : ffff80001025244c: nop : return NULL; 0.00 : ffff800010252450: mov x19, #0x0 // #0 0.00 : ffff800010252454: b ffff800010252360 Percent | Source code & Disassembly of vmlinux for cycles (307 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017afa8 : : tick_program_event(): : : /** : * tick_program_event : */ : int tick_program_event(ktime_t expires, int force) : { 1.62 : ffff80001017afa8: stp x29, x30, [sp, #-48]! : struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); : : if (unlikely(expires == KTIME_MAX)) { 0.00 : ffff80001017afac: mov x2, #0x7fffffffffffffff // #9223372036854775807 : { 0.00 : ffff80001017afb0: mov x29, sp 1.91 : ffff80001017afb4: stp x19, x20, [sp, #16] 0.00 : ffff80001017afb8: mov x20, x0 0.31 : ffff80001017afbc: str x21, [sp, #32] : struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); 0.00 : ffff80001017afc0: adrp x0, ffff800011525000 : { 0.00 : ffff80001017afc4: mov w21, w1 : struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); 0.00 : ffff80001017afc8: add x0, x0, #0xec0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 6.86 : ffff80001017afcc: mrs x1, tpidr_el1 : tick_program_event(): : if (unlikely(expires == KTIME_MAX)) { 1.96 : ffff80001017afd0: cmp x20, x2 : struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); 0.33 : ffff80001017afd4: ldr x19, [x0, x1] : if (unlikely(expires == KTIME_MAX)) { 0.00 : ffff80001017afd8: b.eq ffff80001017b00c // b.none : clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT_STOPPED); : dev->next_event = KTIME_MAX; : return 0; : } : : if (unlikely(clockevent_state_oneshot_stopped(dev))) { 19.84 : ffff80001017afdc: ldr w0, [x19, #56] 0.00 : ffff80001017afe0: cmp w0, #0x4 0.00 : ffff80001017afe4: b.eq ffff80001017b030 // b.none : * before using it. : */ : clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT); : } : : return clockevents_program_event(dev, expires, force); 49.69 : ffff80001017afe8: cmp w21, #0x0 0.00 : ffff80001017afec: mov x1, x20 0.00 : ffff80001017aff0: cset w2, ne // ne = any 0.00 : ffff80001017aff4: mov x0, x19 3.27 : ffff80001017aff8: bl ffff800010178338 : } 9.88 : ffff80001017affc: ldr x21, [sp, #32] 3.98 : ffff80001017b000: ldp x19, x20, [sp, #16] 0.34 : ffff80001017b004: ldp x29, x30, [sp], #48 0.00 : ffff80001017b008: ret : clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT_STOPPED); 0.00 : ffff80001017b00c: mov x0, x19 0.00 : ffff80001017b010: mov w1, #0x4 // #4 0.00 : ffff80001017b014: bl ffff8000101781e0 : dev->next_event = KTIME_MAX; 0.00 : ffff80001017b018: str x20, [x19, #24] : return 0; 0.00 : ffff80001017b01c: mov w0, #0x0 // #0 : } 0.00 : ffff80001017b020: ldp x19, x20, [sp, #16] 0.00 : ffff80001017b024: ldr x21, [sp, #32] 0.00 : ffff80001017b028: ldp x29, x30, [sp], #48 0.00 : ffff80001017b02c: ret : clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT); 0.00 : ffff80001017b030: mov w1, #0x3 // #3 0.00 : ffff80001017b034: mov x0, x19 0.00 : ffff80001017b038: bl ffff8000101781e0 0.00 : ffff80001017b03c: b ffff80001017afe8 Percent | Source code & Disassembly of vmlinux for cycles (298 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015df68 : : rcu_dynticks_eqs_exit(): : * Record exit from an extended quiescent state. This is only to be : * called from an extended quiescent state. : */ : static void rcu_dynticks_eqs_exit(void) : { : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff80001015df68: adrp x0, ffff800011529000 0.00 : ffff80001015df6c: add x0, x0, #0xac0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.34 : ffff80001015df70: mrs x1, tpidr_el1 : rcu_dynticks_eqs_exit(): 0.00 : ffff80001015df74: add x0, x0, x1 : /* : * CPUs seeing atomic_add_return() must see prior idle sojourns, : * and we also must force ordering with the next RCU read-side : * critical section. : */ : seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); 0.00 : ffff80001015df78: add x2, x0, #0xd8 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001015df7c: b ffff80001015df98 2.02 : ffff80001015df80: b ffff80001015df98 : __lse_atomic_add_return(): : } : : ATOMIC_OP_ADD_RETURN(_relaxed, ) : ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") : ATOMIC_OP_ADD_RETURN(_release, l, "memory") : ATOMIC_OP_ADD_RETURN( , al, "memory") 2.01 : ffff80001015df84: mov w1, #0x2 // #2 0.31 : ffff80001015df88: ldaddal w1, w3, [x2] 95.32 : ffff80001015df8c: add w1, w1, w3 : rcu_dynticks_eqs_exit(): : WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && : !(seq & RCU_DYNTICK_CTRL_CTR)); : if (seq & RCU_DYNTICK_CTRL_MASK) { 0.00 : ffff80001015df90: tbnz w1, #0, ffff80001015dfa4 : atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks); : smp_mb__after_atomic(); /* _exit after clearing mask. */ : /* Prefer duplicate flushes to losing a flush. */ : rcu_eqs_special_exit(); : } : } 0.00 : ffff80001015df94: ret : __ll_sc_atomic_add_return(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff80001015df98: add x4, x0, #0xd8 0.00 : ffff80001015df9c: b ffff800010162538 : rcu_dynticks_eqs_exit(): : if (seq & RCU_DYNTICK_CTRL_MASK) { 0.00 : ffff80001015dfa0: tbz w1, #0, ffff80001015df94 : arch_static_branch_jump(): 0.00 : ffff80001015dfa4: b ffff80001015dfc0 0.00 : ffff80001015dfa8: b ffff80001015dfc0 : __lse_atomic_andnot(): : ATOMIC_OP(andnot, stclr) 0.00 : ffff80001015dfac: mov w1, #0x1 // #1 0.00 : ffff80001015dfb0: add x0, x0, #0xd8 0.00 : ffff80001015dfb4: stclr w1, [x0] : rcu_dynticks_eqs_exit(): : smp_mb__after_atomic(); /* _exit after clearing mask. */ 0.00 : ffff80001015dfb8: dmb ish : } 0.00 : ffff80001015dfbc: ret : __ll_sc_atomic_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC_OPS(andnot, bic, ) 0.00 : ffff80001015dfc0: mov w1, #0x1 // #1 0.00 : ffff80001015dfc4: add x0, x0, #0xd8 0.00 : ffff80001015dfc8: b ffff800010162554 : rcu_dynticks_eqs_exit(): : smp_mb__after_atomic(); /* _exit after clearing mask. */ 0.00 : ffff80001015dfcc: dmb ish 0.00 : ffff80001015dfd0: b ffff80001015dfbc Percent | Source code & Disassembly of vmlinux for cycles (290 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016b750 : : hrtimer_start_range_ns(): : * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED); : * softirq based mode is considered for debug purpose only! : */ : void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, : u64 delta_ns, const enum hrtimer_mode mode) : { 0.00 : ffff80001016b750: stp x29, x30, [sp, #-112]! 0.00 : ffff80001016b754: mov x29, sp 10.45 : ffff80001016b758: stp x19, x20, [sp, #16] 0.00 : ffff80001016b75c: mov x19, x0 1.40 : ffff80001016b760: stp x21, x22, [sp, #32] 0.00 : ffff80001016b764: mov w22, w3 0.00 : ffff80001016b768: stp x23, x24, [sp, #48] : * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft : * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard : * expiry mode because unmarked timers are moved to softirq expiry. : */ : if (!IS_ENABLED(CONFIG_PREEMPT_RT)) : WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); 0.00 : ffff80001016b76c: eor x0, x22, #0x4 : { 0.00 : ffff80001016b770: stp x25, x26, [sp, #64] 0.00 : ffff80001016b774: adrp x26, ffff800011909000 0.32 : ffff80001016b778: str x27, [sp, #80] 0.00 : ffff80001016b77c: add x3, x26, #0x908 1.41 : ffff80001016b780: mov x24, x1 : WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); 0.00 : ffff80001016b784: ubfx w0, w0, #2, #1 : { 2.99 : ffff80001016b788: ldr x4, [x3] 2.77 : ffff80001016b78c: str x4, [x29, #104] 0.00 : ffff80001016b790: mov x4, #0x0 // #0 : WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); 4.50 : ffff80001016b794: ldrb w3, [x19, #58] : { 0.00 : ffff80001016b798: mov x23, x2 : WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); 0.00 : ffff80001016b79c: cmp w3, #0x0 0.00 : ffff80001016b7a0: cset w1, eq // eq = none 1.37 : ffff80001016b7a4: cmp w1, w0 0.00 : ffff80001016b7a8: b.ne ffff80001016b9cc // b.any : else : WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard); : : base = lock_hrtimer_base(timer, &flags); 0.00 : ffff80001016b7ac: add x1, x29, #0x60 0.00 : ffff80001016b7b0: mov x0, x19 0.00 : ffff80001016b7b4: bl ffff80001016adc8 1.36 : ffff80001016b7b8: adrp x25, ffff800011525000 : remove_hrtimer(): : u8 state = timer->state; 2.04 : ffff80001016b7bc: ldrb w2, [x19, #56] : hrtimer_start_range_ns(): : base = lock_hrtimer_base(timer, &flags); 0.00 : ffff80001016b7c0: mov x20, x0 : remove_hrtimer(): : if (state & HRTIMER_STATE_ENQUEUED) { 0.00 : ffff80001016b7c4: tbz w2, #0, ffff80001016b7ec : reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016b7c8: ldr x3, [x20] 0.00 : ffff80001016b7cc: add x0, x25, #0x980 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001016b7d0: mrs x1, tpidr_el1 : remove_hrtimer(): 0.00 : ffff80001016b7d4: add x0, x0, x1 : __remove_hrtimer(timer, base, state, reprogram); 0.00 : ffff80001016b7d8: mov x1, x20 : reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016b7dc: cmp x3, x0 : __remove_hrtimer(timer, base, state, reprogram); 0.00 : ffff80001016b7e0: mov x0, x19 0.33 : ffff80001016b7e4: cset w3, eq // eq = none 0.00 : ffff80001016b7e8: bl ffff80001016b130 <__remove_hrtimer> 0.00 : ffff80001016b7ec: mov x21, x24 : __hrtimer_start_range_ns(): : if (mode & HRTIMER_MODE_REL) 0.00 : ffff80001016b7f0: tbz w22, #0, ffff80001016b9c4 : tim = ktime_add_safe(tim, base->get_time()); 0.00 : ffff80001016b7f4: ldr x0, [x20, #48] 0.00 : ffff80001016b7f8: blr x0 : ktime_add_safe(): : ktime_t res = ktime_add_unsafe(lhs, rhs); 0.00 : ffff80001016b7fc: add x21, x24, x0 : if (res < 0 || res < lhs || res < rhs) 0.00 : ffff80001016b800: cmp x21, #0x0 : ktime_t res = ktime_add_unsafe(lhs, rhs); 0.00 : ffff80001016b804: mov x1, x21 : if (res < 0 || res < lhs || res < rhs) 0.00 : ffff80001016b808: ccmp x24, x21, #0x0, ge // ge = tcont 0.00 : ffff80001016b80c: ccmp x0, x21, #0x0, le 0.00 : ffff80001016b810: b.le ffff80001016b81c 0.00 : ffff80001016b814: mov x21, #0x7fffffffffffffff // #9223372036854775807 : res = ktime_set(KTIME_SEC_MAX, 0); 0.00 : ffff80001016b818: mov x1, x21 : if (res < 0 || res < lhs || res < rhs) 2.81 : ffff80001016b81c: adds x21, x23, x21 : res = ktime_set(KTIME_SEC_MAX, 0); 0.00 : ffff80001016b820: mov x0, #0x7fffffffffffffff // #9223372036854775807 : if (res < 0 || res < lhs || res < rhs) 0.00 : ffff80001016b824: ccmp x1, x21, #0x0, pl // pl = nfrst : __hrtimer_start_range_ns(): : new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); 0.00 : ffff80001016b828: and w22, w22, #0x2 : ktime_add_safe(): : res = ktime_set(KTIME_SEC_MAX, 0); 0.00 : ffff80001016b82c: ccmp x23, x21, #0x0, le : switch_hrtimer_base(): : this_cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016b830: add x23, x25, #0x980 : ktime_add_safe(): : res = ktime_set(KTIME_SEC_MAX, 0); 0.00 : ffff80001016b834: csel x21, x21, x0, le : hrtimer_set_expires_range_ns(): : timer->node.expires = ktime_add_safe(time, delta); : } : : static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta) : { : timer->_softexpires = time; 0.00 : ffff80001016b838: stp x21, x1, [x19, #24] : __my_cpu_offset(): 5.13 : ffff80001016b83c: mrs x0, tpidr_el1 : switch_hrtimer_base(): : int basenum = base->index; 1.02 : ffff80001016b840: ldr w24, [x20, #8] : this_cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016b844: add x23, x23, x0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001016b848: b ffff80001016b9b0 : switch_hrtimer_base(): 0.36 : ffff80001016b84c: mov x21, x23 : get_target_base(): : if (static_branch_likely(&timers_migration_enabled) && !pinned) 0.00 : ffff80001016b850: cbnz w22, ffff80001016b86c : return &per_cpu(hrtimer_bases, get_nohz_timer_target()); 0.00 : ffff80001016b854: bl ffff800010114cf8 0.00 : ffff80001016b858: add x25, x25, #0x980 0.00 : ffff80001016b85c: adrp x1, ffff800011909000 0.00 : ffff80001016b860: add x1, x1, #0x928 0.00 : ffff80001016b864: ldr x21, [x1, w0, sxtw #3] 0.00 : ffff80001016b868: add x21, x25, x21 0.00 : ffff80001016b86c: sxtw x24, w24 : switch_hrtimer_base(): : WRITE_ONCE(timer->base, &migration_base); 0.00 : ffff80001016b870: adrp x25, ffff800011925000 0.00 : ffff80001016b874: add x24, x24, #0x1 0.00 : ffff80001016b878: add x25, x25, #0x180 2.45 : ffff80001016b87c: add x25, x25, #0x40 0.00 : ffff80001016b880: lsl x24, x24, #6 : new_base = &new_cpu_base->clock_base[basenum]; 0.00 : ffff80001016b884: add x22, x21, x24 : raw_spin_lock(&new_base->cpu_base->lock); 0.00 : ffff80001016b888: mov x27, x24 : if (base != new_base) { 2.76 : ffff80001016b88c: cmp x20, x22 0.00 : ffff80001016b890: b.eq ffff80001016b900 // b.none 0.00 : ffff80001016b894: nop : hrtimer_callback_running(): : * Helper function to check, whether the timer is running the callback : * function : */ : static inline int hrtimer_callback_running(struct hrtimer *timer) : { : return timer->base->running == timer; 0.00 : ffff80001016b898: ldr x0, [x19, #48] : switch_hrtimer_base(): : if (unlikely(hrtimer_callback_running(timer))) 0.00 : ffff80001016b89c: ldr x0, [x0, #24] 0.00 : ffff80001016b8a0: cmp x19, x0 0.00 : ffff80001016b8a4: b.eq ffff80001016b9a4 // b.none : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001016b8a8: str x25, [x19, #48] : switch_hrtimer_base(): : raw_spin_unlock(&base->cpu_base->lock); 0.00 : ffff80001016b8ac: ldr x0, [x20] 0.00 : ffff80001016b8b0: bl ffff800010cd7c38 <_raw_spin_unlock> : raw_spin_lock(&new_base->cpu_base->lock); 0.00 : ffff80001016b8b4: ldr x0, [x21, x27] 0.00 : ffff80001016b8b8: bl ffff800010cd80f0 <_raw_spin_lock> : if (new_cpu_base != this_cpu_base && 0.00 : ffff80001016b8bc: cmp x23, x21 0.00 : ffff80001016b8c0: b.eq ffff80001016b9b8 // b.none : hrtimer_check_target(timer, new_base)) { 0.00 : ffff80001016b8c4: ldr x0, [x22] : hrtimer_check_target(): : expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); 0.00 : ffff80001016b8c8: ldr x1, [x19, #24] 0.00 : ffff80001016b8cc: ldr x3, [x22, #56] : switch_hrtimer_base(): : if (new_cpu_base != this_cpu_base && 0.00 : ffff80001016b8d0: ldr x2, [x0, #32] : hrtimer_check_target(): : expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); 0.00 : ffff80001016b8d4: sub x1, x1, x3 : switch_hrtimer_base(): : if (new_cpu_base != this_cpu_base && 0.00 : ffff80001016b8d8: cmp x1, x2 0.00 : ffff80001016b8dc: b.ge ffff80001016b9b8 // b.tcont : raw_spin_unlock(&new_base->cpu_base->lock); 0.00 : ffff80001016b8e0: bl ffff800010cd7c38 <_raw_spin_unlock> : raw_spin_lock(&base->cpu_base->lock); 0.00 : ffff80001016b8e4: ldr x0, [x20] 0.00 : ffff80001016b8e8: bl ffff800010cd80f0 <_raw_spin_lock> : __write_once_size(): 0.00 : ffff80001016b8ec: str x20, [x19, #48] : switch_hrtimer_base(): : this_cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016b8f0: mov x21, x23 : new_base = &new_cpu_base->clock_base[basenum]; 0.00 : ffff80001016b8f4: add x22, x21, x24 : if (base != new_base) { 0.00 : ffff80001016b8f8: cmp x20, x22 0.00 : ffff80001016b8fc: b.ne ffff80001016b898 // b.any : if (new_cpu_base != this_cpu_base && 0.33 : ffff80001016b900: cmp x23, x21 7.28 : ffff80001016b904: ldr x2, [x20] 0.00 : ffff80001016b908: b.eq ffff80001016b928 // b.none : hrtimer_check_target(): : expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); 0.00 : ffff80001016b90c: ldr x0, [x19, #24] 0.00 : ffff80001016b910: ldr x3, [x20, #56] : switch_hrtimer_base(): : if (new_cpu_base != this_cpu_base && 0.00 : ffff80001016b914: ldr x1, [x2, #32] : hrtimer_check_target(): : expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); 0.00 : ffff80001016b918: sub x0, x0, x3 : switch_hrtimer_base(): : if (new_cpu_base != this_cpu_base && 0.00 : ffff80001016b91c: cmp x0, x1 0.00 : ffff80001016b920: b.lt ffff80001016b8f0 // b.tstop 0.00 : ffff80001016b924: nop : enqueue_hrtimer(): : base->cpu_base->active_bases |= 1 << base->index; 0.00 : ffff80001016b928: ldr w4, [x22, #8] 0.00 : ffff80001016b92c: mov w20, #0x1 // #1 9.00 : ffff80001016b930: ldr w3, [x2, #8] : return timerqueue_add(&base->active, &timer->node); 0.00 : ffff80001016b934: mov x1, x19 0.00 : ffff80001016b938: add x0, x22, #0x20 : base->cpu_base->active_bases |= 1 << base->index; 0.00 : ffff80001016b93c: lsl w4, w20, w4 13.05 : ffff80001016b940: orr w3, w3, w4 0.00 : ffff80001016b944: str w3, [x2, #8] : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.35 : ffff80001016b948: strb w20, [x19, #56] : enqueue_hrtimer(): : return timerqueue_add(&base->active, &timer->node); 0.00 : ffff80001016b94c: bl ffff800010cc8aa0 : hrtimer_start_range_ns(): : : if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base)) 0.00 : ffff80001016b950: tst w0, #0xff 0.00 : ffff80001016b954: b.eq ffff80001016b964 // b.none : hrtimer_reprogram(timer, true); 2.77 : ffff80001016b958: mov w1, w20 0.00 : ffff80001016b95c: mov x0, x19 0.00 : ffff80001016b960: bl ffff80001016b5b8 : unlock_hrtimer_base(): : raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); 2.72 : ffff80001016b964: ldr x0, [x19, #48] : hrtimer_start_range_ns(): : : unlock_hrtimer_base(timer, &flags); : } 0.00 : ffff80001016b968: add x26, x26, #0x908 : unlock_hrtimer_base(): : raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); 9.34 : ffff80001016b96c: ldr x1, [x29, #96] 8.92 : ffff80001016b970: ldr x0, [x0] 0.00 : ffff80001016b974: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : hrtimer_start_range_ns(): : } 0.00 : ffff80001016b978: ldr x1, [x29, #104] 0.00 : ffff80001016b97c: ldr x0, [x26] 0.00 : ffff80001016b980: eor x0, x1, x0 0.00 : ffff80001016b984: cbnz x0, ffff80001016b9d4 0.00 : ffff80001016b988: ldp x19, x20, [sp, #16] 0.00 : ffff80001016b98c: ldp x21, x22, [sp, #32] 0.67 : ffff80001016b990: ldp x23, x24, [sp, #48] 0.00 : ffff80001016b994: ldp x25, x26, [sp, #64] 0.00 : ffff80001016b998: ldr x27, [sp, #80] 0.00 : ffff80001016b99c: ldp x29, x30, [sp], #112 0.00 : ffff80001016b9a0: ret : switch_hrtimer_base(): : if (unlikely(hrtimer_callback_running(timer))) 0.00 : ffff80001016b9a4: mov x22, x20 0.00 : ffff80001016b9a8: ldr x2, [x20] 0.00 : ffff80001016b9ac: b ffff80001016b928 : this_cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016b9b0: mov x21, x23 0.00 : ffff80001016b9b4: b ffff80001016b86c : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001016b9b8: str x22, [x19, #48] 0.00 : ffff80001016b9bc: ldr x2, [x22] 0.00 : ffff80001016b9c0: b ffff80001016b928 : __hrtimer_start_range_ns(): : if (mode & HRTIMER_MODE_REL) 2.11 : ffff80001016b9c4: mov x1, x24 0.00 : ffff80001016b9c8: b ffff80001016b81c : hrtimer_start_range_ns(): : WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); 0.00 : ffff80001016b9cc: brk #0x800 0.00 : ffff80001016b9d0: b ffff80001016b7ac : } 0.00 : ffff80001016b9d4: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (266 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010167320 : : div_u64_rem(): : * divide. : */ : static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) : { : *remainder = dividend % divisor; : return dividend / divisor; 30.16 : ffff800010167320: mov x1, #0x34db // #13531 0.00 : ffff800010167324: movk x1, #0xd7b6, lsl #16 0.00 : ffff800010167328: movk x1, #0xde82, lsl #32 10.99 : ffff80001016732c: movk x1, #0x431b, lsl #48 58.85 : ffff800010167330: umulh x0, x0, x1 : nsecs_to_jiffies64(): : * Generic case - optimized for cases where HZ is a multiple of 3. : * overflow after 64.99 years, exact for HZ = 60, 72, 90, 120 etc. : */ : return div_u64(n * 9, (9ull * NSEC_PER_SEC + HZ / 2) / HZ); : #endif : } 0.00 : ffff800010167334: lsr x0, x0, #20 0.00 : ffff800010167338: ret Percent | Source code & Disassembly of vmlinux for cycles (390 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012a5a0 : : dequeue_rt_stack(): : /* : * Because the prio of an upper entry depends on the lower : * entries, we must remove entries top - down. : */ : static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags) : { 9.66 : ffff80001012a5a0: stp x29, x30, [sp, #-80]! 0.00 : ffff80001012a5a4: mov x29, sp : struct sched_rt_entity *back = NULL; : : for_each_sched_rt_entity(rt_se) { 0.00 : ffff80001012a5a8: cbz x0, ffff80001012a7a8 1.50 : ffff80001012a5ac: stp x19, x20, [x29, #16] 0.00 : ffff80001012a5b0: mov x19, x0 2.48 : ffff80001012a5b4: stp x21, x22, [x29, #32] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001012a5b8: sub x3, x0, #0x4, lsl #12 1.01 : ffff80001012a5bc: stp x23, x24, [x29, #48] : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012a5c0: adrp x2, ffff800011909000 2.61 : ffff80001012a5c4: stp x25, x26, [x29, #64] 0.00 : ffff80001012a5c8: add x2, x2, #0x928 : dequeue_rt_stack(): : rt_se->back = back; 2.04 : ffff80001012a5cc: str xzr, [x19, #40] : move_entity(): : if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) 0.00 : ffff80001012a5d0: and w21, w1, #0x6 : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012a5d4: adrp x0, ffff800011528000 0.00 : ffff80001012a5d8: add x0, x0, #0xe80 : __read_once_size(): 0.78 : ffff80001012a5dc: ldr w3, [x3, #16004] : rq_of_rt_se(): 0.00 : ffff80001012a5e0: mov x23, x0 0.00 : ffff80001012a5e4: mov x22, x2 : dec_rt_prio(): : rt_rq->highest_prio.curr = MAX_RT_PRIO; 0.00 : ffff80001012a5e8: mov w24, #0x64 // #100 : __clear_bit(): : *p |= mask; : } : : static inline void __clear_bit(int nr, volatile unsigned long *addr) : { : unsigned long mask = BIT_MASK(nr); 1.58 : ffff80001012a5ec: mov x25, #0x1 // #1 : rq_of_rt_se(): : return task_rq(p); 1.84 : ffff80001012a5f0: ldr x1, [x2, w3, uxtw #3] 0.00 : ffff80001012a5f4: add x0, x0, x1 : dequeue_rt_stack(): : back = rt_se; : } : : dequeue_top_rt_rq(rt_rq_of_se(back)); 0.00 : ffff80001012a5f8: add x0, x0, #0x200 0.00 : ffff80001012a5fc: bl ffff800010129378 0.98 : ffff80001012a600: b ffff80001012a60c : : for (rt_se = back; rt_se; rt_se = rt_se->back) { 1.39 : ffff80001012a604: ldr x19, [x19, #40] 0.00 : ffff80001012a608: cbz x19, ffff80001012a6f8 : if (on_rt_rq(rt_se)) 0.79 : ffff80001012a60c: ldrh w0, [x19, #36] 0.00 : ffff80001012a610: cbz w0, ffff80001012a604 : __read_once_size(): 0.52 : ffff80001012a614: sub x1, x19, #0x4, lsl #12 : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012a618: mov x0, x23 : move_entity(): : if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) 0.00 : ffff80001012a61c: cmp w21, #0x2 : __read_once_size(): 0.54 : ffff80001012a620: ldr w1, [x1, #16004] : rq_of_rt_se(): : return task_rq(p); 0.26 : ffff80001012a624: ldr x1, [x22, w1, uxtw #3] 0.00 : ffff80001012a628: add x0, x0, x1 : rt_rq_of_se(): : return &rq->rt; 0.00 : ffff80001012a62c: add x20, x0, #0x200 : move_entity(): : if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) 0.00 : ffff80001012a630: b.eq ffff80001012a73c // b.none : __dequeue_rt_entity(): : WARN_ON_ONCE(!rt_se->on_list); 1.52 : ffff80001012a634: ldrh w1, [x19, #38] 0.00 : ffff80001012a638: cbz w1, ffff80001012a7f0 : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.24 : ffff80001012a63c: ldp x3, x1, [x19] : __list_del(): : next->prev = prev; 0.28 : ffff80001012a640: str x1, [x3, #8] : rt_se_prio(): : return rt_task_of(rt_se)->prio; 0.00 : ffff80001012a644: sub x26, x19, #0x1c0 : __delist_rt_entity(): : if (list_empty(array->queue + rt_se_prio(rt_se))) 0.00 : ffff80001012a648: add x2, x0, #0x210 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012a64c: str x3, [x1] 1.25 : ffff80001012a650: str x19, [x19] : INIT_LIST_HEAD(): : list->prev = list; 0.50 : ffff80001012a654: str x19, [x19, #8] : rt_se_prio(): : return rt_task_of(rt_se)->prio; 0.00 : ffff80001012a658: ldr w1, [x26, #108] : __delist_rt_entity(): : if (list_empty(array->queue + rt_se_prio(rt_se))) 0.00 : ffff80001012a65c: sbfiz x3, x1, #4, #32 0.00 : ffff80001012a660: add x4, x2, x3 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012a664: ldr x2, [x2, x3] : __delist_rt_entity(): 0.00 : ffff80001012a668: cmp x4, x2 0.00 : ffff80001012a66c: b.eq ffff80001012a748 // b.none : rt_se->on_list = 0; 0.00 : ffff80001012a670: strh wzr, [x19, #38] : __dequeue_rt_entity(): : rt_se->on_rq = 0; 0.50 : ffff80001012a674: strh wzr, [x19, #36] : rt_prio(): : : struct task_struct; : : static inline int rt_prio(int prio) : { : if (unlikely(prio < MAX_RT_PRIO)) 0.00 : ffff80001012a678: cmp w1, #0x63 0.00 : ffff80001012a67c: b.le ffff80001012a684 : dec_rt_tasks(): : WARN_ON(!rt_prio(rt_se_prio(rt_se))); 0.00 : ffff80001012a680: brk #0x800 : WARN_ON(!rt_rq->rt_nr_running); 1.82 : ffff80001012a684: ldr w2, [x20, #1616] 0.00 : ffff80001012a688: cbz w2, ffff80001012a7e4 : rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); 1.61 : ffff80001012a68c: sub w2, w2, #0x1 0.51 : ffff80001012a690: str w2, [x20, #1616] : rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se); 2.59 : ffff80001012a694: ldr w1, [x20, #1620] : rt_se_rr_nr_running(): : return (tsk->policy == SCHED_RR) ? 1 : 0; 0.00 : ffff80001012a698: ldr w3, [x26, #728] : dec_rt_prio(): : int prev_prio = rt_rq->highest_prio.curr; 0.78 : ffff80001012a69c: ldr w4, [x20, #1624] : rt_se_rr_nr_running(): : return (tsk->policy == SCHED_RR) ? 1 : 0; 0.00 : ffff80001012a6a0: cmp w3, #0x2 0.00 : ffff80001012a6a4: cset w3, eq // eq = none : dec_rt_tasks(): : rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se); 0.00 : ffff80001012a6a8: sub w1, w1, w3 0.00 : ffff80001012a6ac: str w1, [x20, #1620] : dec_rt_prio(): : if (rt_rq->rt_nr_running) { 0.00 : ffff80001012a6b0: cbz w2, ffff80001012a710 : rt_se_prio(): : return rt_task_of(rt_se)->prio; 0.00 : ffff80001012a6b4: ldr w1, [x26, #108] : dec_rt_prio(): : WARN_ON(prio < prev_prio); 0.00 : ffff80001012a6b8: cmp w1, w4 0.00 : ffff80001012a6bc: b.lt ffff80001012a7f8 // b.tstop : if (prio == prev_prio) { 0.00 : ffff80001012a6c0: b.eq ffff80001012a784 // b.none : dec_rt_migration(): : rt_rq->rt_nr_total--; 0.00 : ffff80001012a6c4: ldr x0, [x20, #1640] 0.00 : ffff80001012a6c8: sub x0, x0, #0x1 0.00 : ffff80001012a6cc: str x0, [x20, #1640] : if (p->nr_cpus_allowed > 1) 0.00 : ffff80001012a6d0: ldr w0, [x26, #732] 0.00 : ffff80001012a6d4: cmp w0, #0x1 0.00 : ffff80001012a6d8: b.le ffff80001012a6e8 : rt_rq->rt_nr_migratory--; 0.00 : ffff80001012a6dc: ldr x0, [x20, #1632] 0.00 : ffff80001012a6e0: sub x0, x0, #0x1 0.00 : ffff80001012a6e4: str x0, [x20, #1632] : update_rt_migration(rt_rq); 1.04 : ffff80001012a6e8: mov x0, x20 0.00 : ffff80001012a6ec: bl ffff80001012a490 : dequeue_rt_stack(): : for (rt_se = back; rt_se; rt_se = rt_se->back) { 0.00 : ffff80001012a6f0: ldr x19, [x19, #40] 0.00 : ffff80001012a6f4: cbnz x19, ffff80001012a60c 0.50 : ffff80001012a6f8: ldp x19, x20, [x29, #16] 6.66 : ffff80001012a6fc: ldp x21, x22, [x29, #32] 2.03 : ffff80001012a700: ldp x23, x24, [x29, #48] 2.16 : ffff80001012a704: ldp x25, x26, [x29, #64] : __dequeue_rt_entity(rt_se, flags); : } : } 0.00 : ffff80001012a708: ldp x29, x30, [sp], #80 0.00 : ffff80001012a70c: ret : dec_rt_prio(): : rt_rq->highest_prio.curr = MAX_RT_PRIO; 4.02 : ffff80001012a710: str w24, [x20, #1624] 6.22 : ffff80001012a714: ldr w1, [x0, #2572] : dec_rt_prio_smp(): : if (rq->online && rt_rq->highest_prio.curr != prev_prio) 0.00 : ffff80001012a718: cbz w1, ffff80001012a6c4 7.45 : ffff80001012a71c: ldr w2, [x20, #1624] 1.02 : ffff80001012a720: cmp w4, w2 0.00 : ffff80001012a724: b.eq ffff80001012a6c4 // b.none : cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr); 5.62 : ffff80001012a728: ldr w1, [x0, #2568] 1.76 : ffff80001012a72c: ldr x0, [x0, #2464] 0.00 : ffff80001012a730: add x0, x0, #0x120 0.00 : ffff80001012a734: bl ffff800010130ab0 0.26 : ffff80001012a738: b ffff80001012a6c4 0.00 : ffff80001012a73c: sub x26, x19, #0x1c0 0.00 : ffff80001012a740: ldr w1, [x26, #108] 0.00 : ffff80001012a744: b ffff80001012a674 : __clear_bit(): : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 6.93 : ffff80001012a748: cmp w1, #0x0 0.00 : ffff80001012a74c: add w3, w1, #0x3f 0.00 : ffff80001012a750: csel w3, w3, w1, lt // lt = tstop : unsigned long mask = BIT_MASK(nr); 0.00 : ffff80001012a754: negs w2, w1 0.78 : ffff80001012a758: and w2, w2, #0x3f 0.00 : ffff80001012a75c: and w1, w1, #0x3f : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.26 : ffff80001012a760: asr w3, w3, #6 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff80001012a764: csneg w1, w1, w2, mi // mi = first : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.52 : ffff80001012a768: sxtw x2, w3 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff80001012a76c: lsl x1, x25, x1 : : *p &= ~mask; 0.78 : ffff80001012a770: ldr x3, [x20, x2, lsl #3] 0.00 : ffff80001012a774: bic x1, x3, x1 12.42 : ffff80001012a778: str x1, [x20, x2, lsl #3] 0.00 : ffff80001012a77c: ldr w1, [x26, #108] 0.00 : ffff80001012a780: b ffff80001012a670 : sched_find_first_bit(): : * one of the 100 bits is cleared. : */ : static inline int sched_find_first_bit(const unsigned long *b) : { : #if BITS_PER_LONG == 64 : if (b[0]) 0.00 : ffff80001012a784: ldr x1, [x0, #512] 0.00 : ffff80001012a788: cbnz x1, ffff80001012a7d8 : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 0.00 : ffff80001012a78c: ldr x1, [x0, #520] 0.00 : ffff80001012a790: rbit x1, x1 0.00 : ffff80001012a794: clz x1, x1 : sched_find_first_bit(): : return __ffs(b[0]); : return __ffs(b[1]) + 64; 0.00 : ffff80001012a798: add w2, w1, #0x40 0.00 : ffff80001012a79c: ldr w1, [x0, #2572] : dec_rt_prio(): : rt_rq->highest_prio.curr = 0.00 : ffff80001012a7a0: str w2, [x20, #1624] 0.00 : ffff80001012a7a4: b ffff80001012a718 : __read_once_size(): 0.00 : ffff80001012a7a8: mov x2, #0xffffffffffffc000 // #-16384 : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012a7ac: adrp x1, ffff800011909000 0.00 : ffff80001012a7b0: add x1, x1, #0x928 0.00 : ffff80001012a7b4: adrp x0, ffff800011528000 : __read_once_size(): 0.00 : ffff80001012a7b8: ldr w2, [x2, #16004] : rq_of_rt_se(): 0.00 : ffff80001012a7bc: add x0, x0, #0xe80 0.00 : ffff80001012a7c0: ldr x1, [x1, w2, uxtw #3] 0.00 : ffff80001012a7c4: add x0, x0, x1 : dequeue_rt_stack(): : dequeue_top_rt_rq(rt_rq_of_se(back)); 0.00 : ffff80001012a7c8: add x0, x0, #0x200 0.00 : ffff80001012a7cc: bl ffff800010129378 : } 0.00 : ffff80001012a7d0: ldp x29, x30, [sp], #80 0.00 : ffff80001012a7d4: ret : __ffs(): 0.00 : ffff80001012a7d8: rbit x2, x1 0.00 : ffff80001012a7dc: clz x2, x2 0.00 : ffff80001012a7e0: b ffff80001012a79c : dec_rt_tasks(): : WARN_ON(!rt_rq->rt_nr_running); 0.00 : ffff80001012a7e4: brk #0x800 0.00 : ffff80001012a7e8: ldr w2, [x20, #1616] 0.00 : ffff80001012a7ec: b ffff80001012a68c : __dequeue_rt_entity(): : WARN_ON_ONCE(!rt_se->on_list); 0.00 : ffff80001012a7f0: brk #0x800 0.00 : ffff80001012a7f4: b ffff80001012a63c : dec_rt_prio(): : WARN_ON(prio < prev_prio); 0.00 : ffff80001012a7f8: brk #0x800 0.00 : ffff80001012a7fc: ldr w1, [x0, #2572] 0.00 : ffff80001012a800: b ffff80001012a718 Percent | Source code & Disassembly of vmlinux for cycles (257 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016a838 : : get_next_timer_interrupt(): : * : * Returns the tick aligned clock monotonic time of the next pending : * timer or KTIME_MAX if no timer is pending. : */ : u64 get_next_timer_interrupt(unsigned long basej, u64 basem) : { 0.00 : ffff80001016a838: stp x29, x30, [sp, #-48]! : : /* : * Pretend that there is no timer pending if the cpu is offline. : * Possible pending timers will be migrated later to an active cpu. : */ : if (cpu_is_offline(smp_processor_id())) 0.00 : ffff80001016a83c: adrp x2, ffff80001151d000 0.81 : ffff80001016a840: add x2, x2, #0x18 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001016a844: adrp x4, ffff80001190a000 <__per_cpu_offset+0x6d8> : get_next_timer_interrupt(): : { 0.40 : ffff80001016a848: mov x29, sp 0.37 : ffff80001016a84c: stp x19, x20, [sp, #16] : test_bit(): 0.00 : ffff80001016a850: add x4, x4, #0x160 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001016a854: mrs x5, tpidr_el1 : get_next_timer_interrupt(): : if (cpu_is_offline(smp_processor_id())) 1.61 : ffff80001016a858: ldr w3, [x2, x5] : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); 0.00 : ffff80001016a85c: adrp x19, ffff800011523000 0.00 : ffff80001016a860: add x19, x19, #0x480 : test_bit(): 0.00 : ffff80001016a864: add w2, w3, #0x3f 4.20 : ffff80001016a868: cmp w3, #0x0 0.00 : ffff80001016a86c: csel w2, w2, w3, lt // lt = tstop 0.00 : ffff80001016a870: asr w2, w2, #6 : get_next_timer_interrupt(): 0.00 : ffff80001016a874: add x19, x19, x5 : test_bit(): 0.40 : ffff80001016a878: sxtw x2, w2 0.73 : ffff80001016a87c: ldr x2, [x4, x2, lsl #3] 3.50 : ffff80001016a880: lsr x3, x2, x3 : get_next_timer_interrupt(): : if (cpu_is_offline(smp_processor_id())) 0.00 : ffff80001016a884: tbz w3, #0, ffff80001016a99c 9.68 : ffff80001016a888: stp x21, x22, [x29, #32] 0.00 : ffff80001016a88c: mov x22, x1 0.00 : ffff80001016a890: mov x21, x0 : return expires; : : raw_spin_lock(&base->lock); 0.00 : ffff80001016a894: mov x0, x19 3.45 : ffff80001016a898: bl ffff800010cd80f0 <_raw_spin_lock> : nextevt = __next_timer_interrupt(base); 0.40 : ffff80001016a89c: mov x0, x19 0.00 : ffff80001016a8a0: bl ffff8000101691d8 <__next_timer_interrupt> : is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA); 2.44 : ffff80001016a8a4: ldr x1, [x19, #16] : base->next_expiry = nextevt; 5.38 : ffff80001016a8a8: str x0, [x19, #24] : /* : * We have a fresh next event. Check whether we can forward the : * base. We can only do that when @basej is past base->clk : * otherwise we might rewind base->clk. : */ : if (time_after(basej, base->clk)) { 0.00 : ffff80001016a8ac: sub x2, x1, x21 0.00 : ffff80001016a8b0: tbnz x2, #63, ffff80001016a984 : base->clk = basej; : else if (time_after(nextevt, base->clk)) : base->clk = nextevt; : } : : if (time_before_eq(nextevt, basej)) { 0.00 : ffff80001016a8b4: sub x2, x21, x0 0.00 : ffff80001016a8b8: tbnz x2, #63, ffff80001016a93c : expires = basem; : base->is_idle = false; 0.00 : ffff80001016a8bc: mov x20, x22 0.00 : ffff80001016a8c0: strb wzr, [x19, #36] : if ((expires - basem) > TICK_NSEC) { : base->must_forward_clk = true; : base->is_idle = true; : } : } : raw_spin_unlock(&base->lock); 13.03 : ffff80001016a8c4: mov x0, x19 0.00 : ffff80001016a8c8: bl ffff800010cd7c38 <_raw_spin_unlock> : cmp_next_hrtimer_event(): : u64 nextevt = hrtimer_get_next_event(); 1.51 : ffff80001016a8cc: bl ffff80001016bb90 : if (expires <= nextevt) 0.00 : ffff80001016a8d0: cmp x20, x0 0.00 : ffff80001016a8d4: b.ls ffff80001016a924 // b.plast : if (nextevt <= basem) 0.00 : ffff80001016a8d8: mov x20, x22 0.00 : ffff80001016a8dc: cmp x0, x22 0.00 : ffff80001016a8e0: b.ls ffff80001016a924 // b.plast : return DIV_ROUND_UP_ULL(nextevt, TICK_NSEC) * TICK_NSEC; 0.00 : ffff80001016a8e4: mov x1, #0x34db // #13531 0.00 : ffff80001016a8e8: add x0, x0, #0x3d0, lsl #12 0.00 : ffff80001016a8ec: movk x1, #0xd7b6, lsl #16 0.00 : ffff80001016a8f0: add x0, x0, #0x8ff 0.00 : ffff80001016a8f4: movk x1, #0xde82, lsl #32 0.00 : ffff80001016a8f8: mov x20, #0x900 // #2304 0.00 : ffff80001016a8fc: movk x1, #0x431b, lsl #48 0.00 : ffff80001016a900: movk x20, #0x3d, lsl #16 0.00 : ffff80001016a904: ldp x21, x22, [x29, #32] 0.00 : ffff80001016a908: umulh x0, x0, x1 0.00 : ffff80001016a90c: lsr x0, x0, #20 0.00 : ffff80001016a910: mul x20, x0, x20 : get_next_timer_interrupt(): : : return cmp_next_hrtimer_event(basem, expires); : } 0.00 : ffff80001016a914: mov x0, x20 0.00 : ffff80001016a918: ldp x19, x20, [sp, #16] 0.00 : ffff80001016a91c: ldp x29, x30, [sp], #48 0.00 : ffff80001016a920: ret 0.00 : ffff80001016a924: mov x0, x20 0.00 : ffff80001016a928: ldp x21, x22, [x29, #32] 0.37 : ffff80001016a92c: ldp x19, x20, [sp, #16] 0.00 : ffff80001016a930: ldp x29, x30, [sp], #48 0.00 : ffff80001016a934: ret : base->clk = basej; 0.00 : ffff80001016a938: str x21, [x19, #16] : is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA); 6.39 : ffff80001016a93c: mov x2, #0x3fffffff // #1073741823 7.46 : ffff80001016a940: add x1, x1, x2 : u64 expires = KTIME_MAX; 0.00 : ffff80001016a944: mov x20, #0x7fffffffffffffff // #9223372036854775807 : if (!is_max_delta) 0.00 : ffff80001016a948: cmp x1, x0 0.38 : ffff80001016a94c: b.eq ffff80001016a960 // b.none : expires = basem + (u64)(nextevt - basej) * TICK_NSEC; 7.93 : ffff80001016a950: sub x0, x0, x21 0.00 : ffff80001016a954: mov x20, #0x900 // #2304 0.00 : ffff80001016a958: movk x20, #0x3d, lsl #16 0.00 : ffff80001016a95c: madd x20, x0, x20, x22 : if ((expires - basem) > TICK_NSEC) { 14.82 : ffff80001016a960: sub x1, x20, x22 0.00 : ffff80001016a964: mov x0, #0x900 // #2304 0.00 : ffff80001016a968: movk x0, #0x3d, lsl #16 0.00 : ffff80001016a96c: cmp x1, x0 0.00 : ffff80001016a970: b.ls ffff80001016a8c4 // b.plast : base->must_forward_clk = true; 0.00 : ffff80001016a974: mov w0, #0x1 // #1 : base->is_idle = true; 14.71 : ffff80001016a978: strb w0, [x19, #36] : base->must_forward_clk = true; 0.00 : ffff80001016a97c: strb w0, [x19, #37] 0.01 : ffff80001016a980: b ffff80001016a8c4 : if (time_after(nextevt, basej)) 0.00 : ffff80001016a984: sub x2, x21, x0 0.00 : ffff80001016a988: tbnz x2, #63, ffff80001016a938 : else if (time_after(nextevt, base->clk)) 0.00 : ffff80001016a98c: sub x1, x1, x0 0.00 : ffff80001016a990: tbz x1, #63, ffff80001016a8bc : base->clk = nextevt; 0.00 : ffff80001016a994: str x0, [x19, #16] 0.00 : ffff80001016a998: b ffff80001016a8bc : return expires; 0.00 : ffff80001016a99c: mov x20, #0x7fffffffffffffff // #9223372036854775807 : } 0.00 : ffff80001016a9a0: mov x0, x20 0.00 : ffff80001016a9a4: ldp x19, x20, [sp, #16] 0.00 : ffff80001016a9a8: ldp x29, x30, [sp], #48 0.00 : ffff80001016a9ac: ret Percent | Source code & Disassembly of vmlinux for cycles (234 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd2a78 : : schedule(): : io_wq_worker_running(tsk); : } : } : : asmlinkage __visible void __sched schedule(void) : { 0.00 : ffff800010cd2a78: stp x29, x30, [sp, #-32]! 0.00 : ffff800010cd2a7c: mov x29, sp 0.90 : ffff800010cd2a80: stp x19, x20, [sp, #16] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 1.28 : ffff800010cd2a84: mrs x20, sp_el0 : sched_submit_work(): : if (!tsk->state) 0.00 : ffff800010cd2a88: ldr x0, [x20, #24] 0.00 : ffff800010cd2a8c: cbz x0, ffff800010cd2ad4 : if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 0.00 : ffff800010cd2a90: ldr w0, [x20, #44] 0.00 : ffff800010cd2a94: and w0, w0, #0x3fffffe0 0.00 : ffff800010cd2a98: and w0, w0, #0xe000003f 0.00 : ffff800010cd2a9c: cbz w0, ffff800010cd2acc : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010cd2aa0: ldr w0, [x20, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010cd2aa4: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010cd2aa8: str w0, [x20, #16] : sched_submit_work(): : if (tsk->flags & PF_WQ_WORKER) 0.00 : ffff800010cd2aac: ldr w0, [x20, #44] 0.00 : ffff800010cd2ab0: tbnz w0, #5, ffff800010cd2b58 : io_wq_worker_sleeping(tsk); 0.00 : ffff800010cd2ab4: mov x0, x20 0.00 : ffff800010cd2ab8: bl ffff8000102eabe8 : get_current(): 0.00 : ffff800010cd2abc: mrs x1, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd2ac0: ldr w0, [x1, #16] : __preempt_count_sub(): : } : : static inline void __preempt_count_sub(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc -= val; 0.00 : ffff800010cd2ac4: sub w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010cd2ac8: str w0, [x1, #16] : sched_submit_work(): : if (tsk_is_pi_blocked(tsk)) 0.00 : ffff800010cd2acc: ldr x0, [x20, #1816] 0.00 : ffff800010cd2ad0: cbz x0, ffff800010cd2b38 : get_current(): 0.00 : ffff800010cd2ad4: mrs x19, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd2ad8: ldr w0, [x19, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff800010cd2adc: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.92 : ffff800010cd2ae0: str w0, [x19, #16] : schedule(): : struct task_struct *tsk = current; : : sched_submit_work(tsk); : do { : preempt_disable(); : __schedule(false); 0.00 : ffff800010cd2ae4: mov w0, #0x0 // #0 0.00 : ffff800010cd2ae8: bl ffff800010cd24e0 <__schedule> : __read_once_size(): : __READ_ONCE_SIZE; 28.96 : ffff800010cd2aec: ldr w0, [x19, #16] : __preempt_count_sub(): : pc -= val; 0.00 : ffff800010cd2af0: sub w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 45.98 : ffff800010cd2af4: str w0, [x19, #16] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010cd2af8: ldr x0, [x19] : schedule(): : sched_preempt_enable_no_resched(); : } while (need_resched()); 0.00 : ffff800010cd2afc: tbnz w0, #1, ffff800010cd2ad4 : sched_update_worker(): : if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 7.65 : ffff800010cd2b00: ldr w1, [x20, #44] 0.00 : ffff800010cd2b04: and w0, w1, #0x3fffffe0 0.00 : ffff800010cd2b08: and w0, w0, #0xe000003f 0.00 : ffff800010cd2b0c: cbz w0, ffff800010cd2b1c : wq_worker_running(tsk); 0.00 : ffff800010cd2b10: mov x0, x20 : if (tsk->flags & PF_WQ_WORKER) 0.00 : ffff800010cd2b14: tbnz w1, #5, ffff800010cd2b28 : io_wq_worker_running(tsk); 0.00 : ffff800010cd2b18: bl ffff8000102eab88 : schedule(): : sched_update_worker(tsk); : } 2.27 : ffff800010cd2b1c: ldp x19, x20, [sp, #16] 7.77 : ffff800010cd2b20: ldp x29, x30, [sp], #32 0.00 : ffff800010cd2b24: ret : sched_update_worker(): : wq_worker_running(tsk); 0.00 : ffff800010cd2b28: bl ffff800010106698 : schedule(): : } 0.00 : ffff800010cd2b2c: ldp x19, x20, [sp, #16] 0.00 : ffff800010cd2b30: ldp x29, x30, [sp], #32 0.00 : ffff800010cd2b34: ret : blk_needs_flush_plug(): : blk_flush_plug_list(plug, true); : } : : static inline bool blk_needs_flush_plug(struct task_struct *tsk) : { : struct blk_plug *plug = tsk->plug; 4.28 : ffff800010cd2b38: ldr x0, [x20, #1840] : : return plug && 0.00 : ffff800010cd2b3c: cbz x0, ffff800010cd2ad4 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd2b40: ldr x1, [x0] : blk_needs_flush_plug(): 0.00 : ffff800010cd2b44: cmp x0, x1 0.00 : ffff800010cd2b48: b.eq ffff800010cd2b64 // b.none : blk_schedule_flush_plug(): : blk_flush_plug_list(plug, true); 0.00 : ffff800010cd2b4c: mov w1, #0x1 // #1 0.00 : ffff800010cd2b50: bl ffff80001045cc00 0.00 : ffff800010cd2b54: b ffff800010cd2ad4 : sched_submit_work(): : wq_worker_sleeping(tsk); 0.00 : ffff800010cd2b58: mov x0, x20 0.00 : ffff800010cd2b5c: bl ffff8000101066f0 0.00 : ffff800010cd2b60: b ffff800010cd2abc : __read_once_size(): 0.00 : ffff800010cd2b64: mov x1, x0 0.00 : ffff800010cd2b68: ldr x2, [x1, #16]! : blk_needs_flush_plug(): : (!list_empty(&plug->mq_list) || 0.00 : ffff800010cd2b6c: cmp x1, x2 0.00 : ffff800010cd2b70: b.ne ffff800010cd2b4c // b.any 0.00 : ffff800010cd2b74: b ffff800010cd2ad4 Percent | Source code & Disassembly of vmlinux for cycles (288 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012bd90 : : pick_next_task_dl(): : : return rb_entry(left, struct sched_dl_entity, rb_node); : } : : static struct task_struct *pick_next_task_dl(struct rq *rq) : { 8.06 : ffff80001012bd90: stp x29, x30, [sp, #-32]! 0.00 : ffff80001012bd94: mov x29, sp 27.24 : ffff80001012bd98: str x20, [sp, #24] : struct sched_dl_entity *dl_se; : struct dl_rq *dl_rq = &rq->dl; : struct task_struct *p; : : if (!sched_dl_runnable(rq)) 1.43 : ffff80001012bd9c: ldr x2, [x0, #2232] 0.74 : ffff80001012bda0: cbz x2, ffff80001012bde8 0.00 : ffff80001012bda4: str x19, [x29, #16] 0.00 : ffff80001012bda8: mov x19, x0 : return NULL; : : dl_se = pick_next_dl_entity(rq, dl_rq); 0.00 : ffff80001012bdac: ldr x2, [x0, #2224] : pick_next_dl_entity(): : if (!left) 0.00 : ffff80001012bdb0: cbz x2, ffff80001012bde4 : set_next_task_dl(): : p->se.exec_start = rq_clock_task(rq); 0.00 : ffff80001012bdb4: ldr x3, [x0, #2432] : dl_task_of(): : return container_of(dl_se, struct task_struct, dl); 0.00 : ffff80001012bdb8: sub x20, x2, #0x1f8 : set_next_task_dl(): : dequeue_pushable_dl_task(rq, p); 0.00 : ffff80001012bdbc: mov x1, x20 : p->se.exec_start = rq_clock_task(rq); 0.00 : ffff80001012bdc0: stur x3, [x2, #-240] : dequeue_pushable_dl_task(rq, p); 0.00 : ffff80001012bdc4: bl ffff80001012bc08 0.00 : ffff80001012bdc8: mov x0, x19 0.00 : ffff80001012bdcc: bl ffff80001012bcb8 : pick_next_task_dl(): : BUG_ON(!dl_se); : p = dl_task_of(dl_se); : set_next_task_dl(rq, p, true); : return p; 0.00 : ffff80001012bdd0: ldr x19, [x29, #16] : } 2.06 : ffff80001012bdd4: mov x0, x20 13.48 : ffff80001012bdd8: ldr x20, [sp, #24] 40.30 : ffff80001012bddc: ldp x29, x30, [sp], #32 0.00 : ffff80001012bde0: ret : BUG_ON(!dl_se); 0.00 : ffff80001012bde4: brk #0x800 : return NULL; 6.70 : ffff80001012bde8: mov x20, #0x0 // #0 0.00 : ffff80001012bdec: b ffff80001012bdd4 Percent | Source code & Disassembly of vmlinux for cycles (514 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010716ea8 : : arm_lpae_map(): : int ret, lvl = data->start_level; : arm_lpae_iopte prot; : long iaext = (long)iova >> cfg->ias; : : /* If no access, then nothing to do */ : if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) 0.40 : ffff800010716ea8: and w6, w4, #0x3 : { 0.00 : ffff800010716eac: mov x5, x0 : if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) 0.00 : ffff800010716eb0: mov w0, w6 0.00 : ffff800010716eb4: cbz w6, ffff800010716f04 : { 0.20 : ffff800010716eb8: stp x29, x30, [sp, #-16]! 0.00 : ffff800010716ebc: mov x29, sp : return 0; : : if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) 0.00 : ffff800010716ec0: cbnz x3, ffff800010716ed4 0.00 : ffff800010716ec4: brk #0x800 : return -EINVAL; 0.00 : ffff800010716ec8: mov w0, #0xffffffea // #-22 : * a chance for anything to kick off a table walk for the new iova. : */ : wmb(); : : return ret; : } 0.00 : ffff800010716ecc: ldp x29, x30, [sp], #16 0.00 : ffff800010716ed0: ret : if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) 0.58 : ffff800010716ed4: ldur x0, [x5, #-64] 0.00 : ffff800010716ed8: bics xzr, x3, x0 0.00 : ffff800010716edc: b.ne ffff800010716ec4 // b.any : if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 0.20 : ffff800010716ee0: ldur x9, [x5, #-72] : long iaext = (long)iova >> cfg->ias; 0.00 : ffff800010716ee4: ldur w0, [x5, #-56] : iaext = ~iaext; 0.00 : ffff800010716ee8: tst x9, #0x20 : long iaext = (long)iova >> cfg->ias; 0.00 : ffff800010716eec: asr x0, x1, x0 : iaext = ~iaext; 0.00 : ffff800010716ef0: cinv x0, x0, ne // ne = any : if (WARN_ON(iaext || paddr >> cfg->oas)) 0.00 : ffff800010716ef4: cbz x0, ffff800010716f08 0.00 : ffff800010716ef8: brk #0x800 : return -ERANGE; 0.00 : ffff800010716efc: mov w0, #0xffffffde // #-34 0.00 : ffff800010716f00: b ffff800010716ecc : } 0.00 : ffff800010716f04: ret : if (WARN_ON(iaext || paddr >> cfg->oas)) 0.40 : ffff800010716f08: ldur w0, [x5, #-52] 0.00 : ffff800010716f0c: lsr x0, x2, x0 0.00 : ffff800010716f10: cbnz x0, ffff800010716ef8 : prot = arm_lpae_prot_to_pte(data, iommu_prot); 0.00 : ffff800010716f14: ldur w8, [x5, #-88] : arm_lpae_prot_to_pte(): : if (data->iop.fmt == ARM_64_LPAE_S1 || 0.00 : ffff800010716f18: ands w10, w8, #0xfffffffd 0.00 : ffff800010716f1c: b.ne ffff800010716f58 // b.any : pte = ARM_LPAE_PTE_nG; 0.38 : ffff800010716f20: cmp w6, #0x1 0.00 : ffff800010716f24: mov x0, #0x800 // #2048 0.40 : ffff800010716f28: mov x7, #0x880 // #2176 0.00 : ffff800010716f2c: csel x7, x7, x0, eq // eq = none : pte |= ARM_LPAE_PTE_AP_UNPRIV; 0.00 : ffff800010716f30: orr x0, x7, #0x40 0.00 : ffff800010716f34: tst x4, #0x20 0.00 : ffff800010716f38: csel x7, x0, x7, eq // eq = none 0.00 : ffff800010716f3c: and w6, w4, #0x10 0.00 : ffff800010716f40: and w0, w4, #0x4 : if (prot & IOMMU_MMIO) 0.00 : ffff800010716f44: cbnz w6, ffff800010716fcc : else if (prot & IOMMU_CACHE) 0.40 : ffff800010716f48: cbz w0, ffff800010716fe4 : pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE 0.00 : ffff800010716f4c: orr x7, x7, #0x4 : pte |= ARM_LPAE_PTE_SH_IS; 0.00 : ffff800010716f50: orr x7, x7, #0x300 0.00 : ffff800010716f54: b ffff800010716f8c : pte |= ARM_LPAE_PTE_HAP_WRITE; 0.00 : ffff800010716f58: ubfiz x7, x4, #6, #2 0.00 : ffff800010716f5c: and w6, w4, #0x10 0.00 : ffff800010716f60: and w0, w4, #0x4 : if (data->iop.fmt == ARM_64_LPAE_S2 || 0.00 : ffff800010716f64: cmp w10, #0x1 0.00 : ffff800010716f68: b.ne ffff800010716f44 // b.any : if (prot & IOMMU_MMIO) 0.00 : ffff800010716f6c: cbnz w6, ffff800010716f80 : else if (prot & IOMMU_CACHE) 0.00 : ffff800010716f70: cbz w0, ffff800010716fd8 : pte |= ARM_LPAE_PTE_MEMATTR_OIWB; 0.00 : ffff800010716f74: orr x7, x7, #0x3c : pte |= ARM_LPAE_PTE_SH_IS; 0.00 : ffff800010716f78: orr x7, x7, #0x300 0.00 : ffff800010716f7c: b ffff800010716f8c : pte |= ARM_LPAE_PTE_MEMATTR_DEV; 0.00 : ffff800010716f80: orr x7, x7, #0x4 : if (prot & IOMMU_CACHE) 0.00 : ffff800010716f84: cbnz w0, ffff800010716f50 : pte |= ARM_LPAE_PTE_SH_OS; 0.00 : ffff800010716f88: orr x7, x7, #0x200 : arm_lpae_map(): : arm_lpae_iopte *ptep = data->pgd; 0.38 : ffff800010716f8c: sub x0, x5, #0x58 : arm_lpae_prot_to_pte(): : pte |= ARM_LPAE_PTE_XN; 0.00 : ffff800010716f90: tst x4, #0x8 0.00 : ffff800010716f94: orr x4, x7, #0x60000000000000 : arm_lpae_map(): : ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); 0.00 : ffff800010716f98: ldr x6, [x0, #128] : arm_lpae_prot_to_pte(): : pte |= ARM_LPAE_PTE_XN; 0.00 : ffff800010716f9c: csel x7, x4, x7, ne // ne = any : pte |= ARM_LPAE_PTE_NS; 0.20 : ffff800010716fa0: tst x9, #0x1 0.00 : ffff800010716fa4: orr x4, x7, #0x20 : arm_lpae_map(): : ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); 0.55 : ffff800010716fa8: ldr w5, [x0, #116] : arm_lpae_prot_to_pte(): : pte |= ARM_LPAE_PTE_NS; 0.00 : ffff800010716fac: csel x7, x4, x7, ne // ne = any : pte |= ARM_LPAE_PTE_AF; 0.00 : ffff800010716fb0: cmp w8, #0x5 0.00 : ffff800010716fb4: orr x4, x7, #0x400 : arm_lpae_map(): : ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); 0.00 : ffff800010716fb8: csel x4, x4, x7, ne // ne = any 0.00 : ffff800010716fbc: bl ffff800010716be0 <__arm_lpae_map> : wmb(); 0.00 : ffff800010716fc0: dsb st : } 95.92 : ffff800010716fc4: ldp x29, x30, [sp], #16 0.00 : ffff800010716fc8: ret : arm_lpae_prot_to_pte(): : pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV 0.00 : ffff800010716fcc: orr x7, x7, #0x8 : if (prot & IOMMU_CACHE) 0.00 : ffff800010716fd0: cbz w0, ffff800010716f88 0.00 : ffff800010716fd4: b ffff800010716f50 : pte |= ARM_LPAE_PTE_MEMATTR_NC; 0.00 : ffff800010716fd8: mov x0, #0x14 // #20 0.00 : ffff800010716fdc: orr x7, x7, x0 0.00 : ffff800010716fe0: b ffff800010716f88 : pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 0.00 : ffff800010716fe4: orr x0, x7, #0xc 0.00 : ffff800010716fe8: tst x4, #0x40 0.00 : ffff800010716fec: csel x7, x0, x7, ne // ne = any 0.00 : ffff800010716ff0: b ffff800010716f88 Percent | Source code & Disassembly of vmlinux for cycles (216 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010712818 : : iommu_get_dma_domain(): : * For IOMMU_DOMAIN_DMA implementations which already provide their own : * guarantees that the group and its default domain are valid and correct. : */ : struct iommu_domain *iommu_get_dma_domain(struct device *dev) : { : return dev->iommu_group->default_domain; 33.99 : ffff800010712818: ldr x0, [x0, #736] : } 57.41 : ffff80001071281c: ldr x0, [x0, #200] 8.60 : ffff800010712820: ret Percent | Source code & Disassembly of vmlinux for cycles (279 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010134c40 : : update_rt_rq_load_avg(): : * load_avg and runnable_load_avg are not supported and meaningless. : * : */ : : int update_rt_rq_load_avg(u64 now, struct rq *rq, int running) : { 0.00 : ffff800010134c40: mov w9, w2 : ___update_load_sum(): : delta = now - sa->last_update_time; 0.00 : ffff800010134c44: ldr x2, [x1, #2624] : update_rt_rq_load_avg(): : { 0.00 : ffff800010134c48: mov x8, x1 : ___update_load_sum(): : delta = now - sa->last_update_time; 0.00 : ffff800010134c4c: add x7, x1, #0xa40 : if ((s64)delta < 0) { 0.00 : ffff800010134c50: subs x1, x0, x2 4.75 : ffff800010134c54: b.mi ffff800010134d58 // b.first : delta >>= 10; 4.26 : ffff800010134c58: lsr x3, x1, #10 : if (!delta) 0.00 : ffff800010134c5c: cbz x3, ffff800010134d50 : update_rt_rq_load_avg(): : { 2.12 : ffff800010134c60: stp x29, x30, [sp, #-16]! : ___update_load_sum(): : sa->last_update_time += delta << 10; 0.00 : ffff800010134c64: and x1, x1, #0xfffffffffffffc00 0.00 : ffff800010134c68: add x1, x1, x2 0.00 : ffff800010134c6c: mov w0, w3 : update_rt_rq_load_avg(): : { 7.51 : ffff800010134c70: mov x29, sp : ___update_load_sum(): : sa->last_update_time += delta << 10; 0.00 : ffff800010134c74: str x1, [x8, #2624] : update_rt_rq_load_avg(): : if (___update_load_sum(now, &rq->avg_rt, 0.00 : ffff800010134c78: sxtw x11, w9 0.00 : ffff800010134c7c: ldr w1, [x7, #28] 0.39 : ffff800010134c80: add x3, x3, w1, uxtw 0.00 : ffff800010134c84: lsr x10, x3, #10 : ___update_load_sum(): : if (!load) 0.00 : ffff800010134c88: cbnz x11, ffff800010134d64 : runnable = running = 0; 1.08 : ffff800010134c8c: mov x13, #0x0 // #0 0.00 : ffff800010134c90: mov w9, #0x0 // #0 : accumulate_sum(): : if (periods) { 0.00 : ffff800010134c94: cbz x10, ffff800010134d88 : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.75 : ffff800010134c98: cmp x10, #0x7e0 : accumulate_sum(): : sa->load_sum = decay_load(sa->load_sum, periods); 1.04 : ffff800010134c9c: ldr x4, [x7, #8] : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.37 : ffff800010134ca0: b.hi ffff800010134dbc // b.pmore : if (unlikely(local_n >= LOAD_AVG_PERIOD)) { 2.91 : ffff800010134ca4: cmp x10, #0x1f 0.00 : ffff800010134ca8: ldr w12, [x7, #24] 0.37 : ffff800010134cac: ldr x5, [x7, #16] 0.00 : ffff800010134cb0: b.hi ffff800010134dc8 // b.pmore : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010134cb4: adrp x6, ffff800010d06000 0.00 : ffff800010134cb8: add x6, x6, #0x450 : mul_u64_u32_shr(): : #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) : : #ifndef mul_u64_u32_shr : static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) : { : return (u64)(((unsigned __int128)a * mul) >> shift); 0.39 : ffff800010134cbc: ldr w2, [x6, w10, uxtw #2] 0.00 : ffff800010134cc0: mul x14, x2, x4 0.00 : ffff800010134cc4: mul x6, x2, x5 0.00 : ffff800010134cc8: umulh x4, x2, x4 1.87 : ffff800010134ccc: umulh x5, x2, x5 0.00 : ffff800010134cd0: extr x4, x4, x14, #32 0.00 : ffff800010134cd4: extr x5, x5, x6, #32 0.00 : ffff800010134cd8: stp x4, x5, [x7, #8] 0.00 : ffff800010134cdc: mul x2, x12, x2 0.00 : ffff800010134ce0: lsr x2, x2, #32 : accumulate_sum(): : sa->util_sum = decay_load((u64)(sa->util_sum), periods); 0.00 : ffff800010134ce4: str w2, [x7, #24] : delta %= 1024; 0.00 : ffff800010134ce8: and x14, x3, #0x3ff 0.00 : ffff800010134cec: mov w12, w14 : if (load) { 0.00 : ffff800010134cf0: cbnz x11, ffff800010134da0 : sa->period_contrib = delta; 0.00 : ffff800010134cf4: str w14, [x7, #28] : if (runnable) 1.05 : ffff800010134cf8: cbz x13, ffff800010134d0c : sa->runnable_load_sum += runnable * contrib; 7.05 : ffff800010134cfc: ldr x2, [x7, #16] 14.57 : ffff800010134d00: mov w1, w0 0.00 : ffff800010134d04: madd x13, x1, x13, x2 2.21 : ffff800010134d08: str x13, [x7, #16] : if (running) 0.00 : ffff800010134d0c: cbz w9, ffff800010134d1c : sa->util_sum += contrib << SCHED_CAPACITY_SHIFT; 12.14 : ffff800010134d10: ldr w1, [x7, #24] 0.00 : ffff800010134d14: add w0, w1, w0, lsl #10 3.54 : ffff800010134d18: str w0, [x7, #24] : ___update_load_sum(): : if (!accumulate_sum(delta, sa, load, runnable, running)) 0.00 : ffff800010134d1c: cbz w10, ffff800010134d8c : div_u64_rem(): : return dividend / divisor; 0.00 : ffff800010134d20: ldp x4, x3, [x7, #8] : ___update_load_avg(): : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010134d24: mov w0, #0xb67e // #46718 : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 3.31 : ffff800010134d28: ldr w2, [x7, #24] : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010134d2c: add w5, w12, w0 : update_rt_rq_load_avg(): : running, : running)) { : : ___update_load_avg(&rq->avg_rt, 1, 1); : trace_pelt_rt_tp(rq); : return 1; 0.00 : ffff800010134d30: mov w0, #0x1 // #1 : div_u64_rem(): 0.00 : ffff800010134d34: udiv x4, x4, x5 0.37 : ffff800010134d38: udiv x3, x3, x5 : ___update_load_avg(): : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 0.00 : ffff800010134d3c: udiv w1, w2, w5 : sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider); 0.00 : ffff800010134d40: stp x4, x3, [x7, #32] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010134d44: str x1, [x8, #2672] : update_rt_rq_load_avg(): : } : : return 0; : } 0.00 : ffff800010134d48: ldp x29, x30, [sp], #16 0.00 : ffff800010134d4c: ret : return 0; 0.00 : ffff800010134d50: mov w0, #0x0 // #0 : } 0.00 : ffff800010134d54: ret : ___update_load_sum(): : sa->last_update_time = now; 0.00 : ffff800010134d58: str x0, [x8, #2624] : update_rt_rq_load_avg(): : return 0; 0.00 : ffff800010134d5c: mov w0, #0x0 // #0 0.00 : ffff800010134d60: ret : accumulate_sum(): : if (periods) { 0.74 : ffff800010134d64: cbnz x10, ffff800010134d98 : sa->period_contrib = delta; 1.73 : ffff800010134d68: mov w12, w3 0.00 : ffff800010134d6c: mov x13, x11 5.69 : ffff800010134d70: str w3, [x7, #28] : sa->load_sum += load * contrib; 0.76 : ffff800010134d74: ldr x2, [x7, #8] 0.00 : ffff800010134d78: mov w1, w0 0.00 : ffff800010134d7c: madd x11, x1, x11, x2 2.98 : ffff800010134d80: str x11, [x7, #8] 0.00 : ffff800010134d84: b ffff800010134cf8 : sa->period_contrib = delta; 0.33 : ffff800010134d88: str w3, [x7, #28] : update_rt_rq_load_avg(): : return 0; 0.00 : ffff800010134d8c: mov w0, #0x0 // #0 : } 13.60 : ffff800010134d90: ldp x29, x30, [sp], #16 0.00 : ffff800010134d94: ret : accumulate_sum(): : if (periods) { 2.13 : ffff800010134d98: mov x13, x11 0.00 : ffff800010134d9c: b ffff800010134c98 : contrib = __accumulate_pelt_segments(periods, 0.00 : ffff800010134da0: mov w0, #0x400 // #1024 0.00 : ffff800010134da4: mov w2, w14 0.00 : ffff800010134da8: sub w1, w0, w1 0.00 : ffff800010134dac: mov x0, x10 0.00 : ffff800010134db0: bl ffff8000101346b8 <__accumulate_pelt_segments> : sa->period_contrib = delta; 0.00 : ffff800010134db4: str w14, [x7, #28] 0.00 : ffff800010134db8: b ffff800010134d74 : sa->runnable_load_sum = 0.00 : ffff800010134dbc: mov w2, #0x0 // #0 0.00 : ffff800010134dc0: stp xzr, xzr, [x7, #8] 0.00 : ffff800010134dc4: b ffff800010134ce4 : decay_load(): : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010134dc8: and x14, x10, #0x1f 0.00 : ffff800010134dcc: adrp x2, ffff800010d06000 0.00 : ffff800010134dd0: add x2, x2, #0x450 : val >>= local_n / LOAD_AVG_PERIOD; 0.00 : ffff800010134dd4: lsr w6, w10, #5 0.00 : ffff800010134dd8: lsr x4, x4, x6 : mul_u64_u32_shr(): : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff800010134ddc: ldr w2, [x2, x14, lsl #2] : decay_load(): 0.00 : ffff800010134de0: lsr x5, x5, x6 0.00 : ffff800010134de4: lsr x12, x12, x6 : mul_u64_u32_shr(): 0.00 : ffff800010134de8: mul x14, x4, x2 0.00 : ffff800010134dec: umulh x4, x4, x2 0.00 : ffff800010134df0: mul x6, x5, x2 0.00 : ffff800010134df4: umulh x5, x5, x2 0.00 : ffff800010134df8: extr x4, x4, x14, #32 0.00 : ffff800010134dfc: str x4, [x7, #8] 0.00 : ffff800010134e00: extr x4, x5, x6, #32 0.00 : ffff800010134e04: str x4, [x7, #16] 0.00 : ffff800010134e08: b ffff800010134cdc Percent | Source code & Disassembly of vmlinux for cycles (217 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010127a00 : : pick_next_task_fair(): : set_last_buddy(se); : } : : struct task_struct * : pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) : { 0.00 : ffff800010127a00: stp x29, x30, [sp, #-96]! 0.00 : ffff800010127a04: mov x29, sp 26.42 : ffff800010127a08: stp x19, x20, [sp, #16] 0.00 : ffff800010127a0c: mov x19, x0 2.25 : ffff800010127a10: str x23, [sp, #48] 0.00 : ffff800010127a14: mov x20, x2 0.83 : ffff800010127a18: str x21, [sp, #32] 0.00 : ffff800010127a1c: mov x23, x1 0.00 : ffff800010127a20: b ffff800010127a38 : update_misfit_status(p, rq); : : return p; : : idle: : if (!rf) 4.58 : ffff800010127a24: cbz x20, ffff800010127b0c : return NULL; : : new_tasks = newidle_balance(rq, rf); 0.00 : ffff800010127a28: bl ffff800010127620 : /* : * Because newidle_balance() releases (and re-acquires) rq->lock, it is : * possible for any higher priority task to appear. In that case we : * must re-start the pick_next_entity() loop. : */ : if (new_tasks < 0) 0.00 : ffff800010127a2c: cmp w0, #0x0 0.00 : ffff800010127a30: b.lt ffff800010127b28 // b.tstop : return RETRY_TASK; : : if (new_tasks > 0) 0.00 : ffff800010127a34: b.eq ffff800010127ae8 // b.none : if (!sched_fair_runnable(rq)) 0.93 : ffff800010127a38: ldr w2, [x19, #152] : new_tasks = newidle_balance(rq, rf); 0.00 : ffff800010127a3c: mov x1, x20 3.23 : ffff800010127a40: mov x0, x19 : if (!sched_fair_runnable(rq)) 0.00 : ffff800010127a44: cbz w2, ffff800010127a24 : struct cfs_rq *cfs_rq = &rq->cfs; 0.00 : ffff800010127a48: add x20, x19, #0x80 : if (!prev || prev->sched_class != &fair_sched_class) 0.00 : ffff800010127a4c: cbz x23, ffff800010127a80 0.00 : ffff800010127a50: ldr x1, [x23, #128] 0.00 : ffff800010127a54: adrp x0, ffff800010d06000 0.00 : ffff800010127a58: add x0, x0, #0x128 0.00 : ffff800010127a5c: cmp x1, x0 0.00 : ffff800010127a60: b.eq ffff800010127bd4 // b.none : put_prev_task(): : #endif : }; : : static inline void put_prev_task(struct rq *rq, struct task_struct *prev) : { : WARN_ON_ONCE(rq->curr != prev); 0.00 : ffff800010127a64: ldr x0, [x19, #2352] 0.00 : ffff800010127a68: cmp x23, x0 0.00 : ffff800010127a6c: b.ne ffff800010127cec // b.any : prev->sched_class->put_prev_task(rq, prev); 0.00 : ffff800010127a70: ldr x2, [x1, #56] 0.00 : ffff800010127a74: mov x0, x19 0.00 : ffff800010127a78: mov x1, x23 0.00 : ffff800010127a7c: blr x2 : pick_next_task_fair(): : se = pick_next_entity(cfs_rq, NULL); 0.49 : ffff800010127a80: mov x0, x20 0.00 : ffff800010127a84: mov x1, #0x0 // #0 0.00 : ffff800010127a88: bl ffff80001011ddd0 0.00 : ffff800010127a8c: mov x21, x0 : set_next_entity(cfs_rq, se); 0.00 : ffff800010127a90: mov x1, x0 0.00 : ffff800010127a94: mov x0, x20 0.00 : ffff800010127a98: bl ffff80001011f9c0 : cfs_rq = group_cfs_rq(se); 0.00 : ffff800010127a9c: ldr x20, [x21, #136] : } while (cfs_rq); 0.00 : ffff800010127aa0: cbnz x20, ffff800010127a80 : task_of(): : return container_of(se, struct task_struct, se); 0.99 : ffff800010127aa4: sub x21, x21, #0xc0 : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff800010127aa8: ldp x3, x2, [x21, #240] : __list_del(): : next->prev = prev; 0.00 : ffff800010127aac: str x2, [x3, #8] : pick_next_task_fair(): : list_move(&p->se.group_node, &rq->cfs_tasks); 0.00 : ffff800010127ab0: add x0, x21, #0xf0 0.00 : ffff800010127ab4: add x1, x19, #0xa10 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 2.49 : ffff800010127ab8: str x3, [x2] : list_add(): : __list_add(new, head, head->next); 1.51 : ffff800010127abc: ldr x2, [x19, #2576] : __list_add(): : next->prev = new; 3.87 : ffff800010127ac0: str x0, [x2, #8] : new->prev = prev; 0.50 : ffff800010127ac4: stp x2, x1, [x21, #240] : __write_once_size(): 0.00 : ffff800010127ac8: str x0, [x19, #2576] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010127acc: nop : pick_next_task_fair(): : * lost_idle_time of clock_pelt : */ : update_idle_rq_clock_pelt(rq); : : return NULL; : } 0.00 : ffff800010127ad0: mov x0, x21 0.00 : ffff800010127ad4: ldr x21, [sp, #32] 0.00 : ffff800010127ad8: ldp x19, x20, [sp, #16] 0.00 : ffff800010127adc: ldr x23, [sp, #48] 0.00 : ffff800010127ae0: ldp x29, x30, [sp], #96 0.00 : ffff800010127ae4: ret : update_idle_rq_clock_pelt(): : * bound of util_sum to decide. : */ : static inline void update_idle_rq_clock_pelt(struct rq *rq) : { : u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; : u32 util_sum = rq->cfs.avg.util_sum; 0.00 : ffff800010127ae8: ldr w0, [x19, #280] : * considered as an always runnig rq without idle time to : * steal. This potential idle time is considered as lost in : * this case. We keep track of this lost idle time compare to : * rq's clock_task. : */ : if (util_sum >= divider) 0.00 : ffff800010127aec: mov w1, #0x3d81 // #15745 : util_sum += rq->avg_rt.util_sum; 0.00 : ffff800010127af0: ldr w3, [x19, #2648] : if (util_sum >= divider) 0.00 : ffff800010127af4: movk w1, #0x2d9, lsl #16 : util_sum += rq->avg_dl.util_sum; 0.00 : ffff800010127af8: ldr w2, [x19, #2712] : util_sum += rq->avg_rt.util_sum; 0.00 : ffff800010127afc: add w0, w0, w3 : util_sum += rq->avg_dl.util_sum; 0.00 : ffff800010127b00: add w0, w0, w2 : if (util_sum >= divider) 0.00 : ffff800010127b04: cmp w0, w1 0.00 : ffff800010127b08: b.hi ffff800010127ba0 // b.pmore : pick_next_task_fair(): : return NULL; 15.87 : ffff800010127b0c: mov x21, #0x0 // #0 : } 0.00 : ffff800010127b10: mov x0, x21 4.51 : ffff800010127b14: ldr x21, [sp, #32] 28.75 : ffff800010127b18: ldp x19, x20, [sp, #16] 2.76 : ffff800010127b1c: ldr x23, [sp, #48] 0.00 : ffff800010127b20: ldp x29, x30, [sp], #96 0.00 : ffff800010127b24: ret : return RETRY_TASK; 0.00 : ffff800010127b28: mov x21, #0xffffffffffffffff // #-1 : } 0.00 : ffff800010127b2c: ldr x23, [sp, #48] 0.00 : ffff800010127b30: mov x0, x21 0.00 : ffff800010127b34: ldr x21, [sp, #32] 0.00 : ffff800010127b38: ldp x19, x20, [sp, #16] 0.00 : ffff800010127b3c: ldp x29, x30, [sp], #96 0.00 : ffff800010127b40: ret : update_misfit_status(): : if (!p) { 0.00 : ffff800010127b44: cbz x21, ffff800010127b98 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010127b48: ldrsw x5, [x19, #2568] 0.00 : ffff800010127b4c: adrp x2, ffff800011909000 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010127b50: ldr x4, [x21, #432] : capacity_of(): 0.00 : ffff800010127b54: add x2, x2, #0x928 : __read_once_size(): 0.00 : ffff800010127b58: ldr x3, [x21, #440] : capacity_of(): 0.00 : ffff800010127b5c: adrp x1, ffff800011528000 0.00 : ffff800010127b60: add x1, x1, #0xe80 0.00 : ffff800010127b64: ldr x2, [x2, x5, lsl #3] : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff800010127b68: lsr x0, x3, #32 0.00 : ffff800010127b6c: cmp w0, w3 0.00 : ffff800010127b70: csel w0, w0, w3, hi // hi = pmore : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010127b74: add x1, x2, x1 : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff800010127b78: orr w0, w0, #0x1 : task_util_est(): : return max(task_util(p), _task_util_est(p)); 0.00 : ffff800010127b7c: cmp x0, x4 0.00 : ffff800010127b80: csel x0, x0, x4, cs // cs = hs, nlast : task_fits_capacity(): : return fits_capacity(uclamp_task_util(p), capacity); 0.00 : ffff800010127b84: ldr x1, [x1, #2480] 0.00 : ffff800010127b88: add x0, x0, x0, lsl #2 0.00 : ffff800010127b8c: lsl x1, x1, #10 : update_misfit_status(): : if (task_fits_capacity(p, capacity_of(cpu_of(rq)))) { 0.00 : ffff800010127b90: cmp x1, x0, lsl #8 0.00 : ffff800010127b94: b.ls ffff800010127ca4 // b.plast : rq->misfit_task_load = 0; 0.00 : ffff800010127b98: str xzr, [x19, #2512] 0.00 : ffff800010127b9c: b ffff800010127b10 : update_idle_rq_clock_pelt(): : rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; 0.00 : ffff800010127ba0: ldr x0, [x19, #2432] : pick_next_task_fair(): : return NULL; 0.00 : ffff800010127ba4: mov x21, #0x0 // #0 : update_idle_rq_clock_pelt(): 0.00 : ffff800010127ba8: ldr x2, [x19, #2448] 0.00 : ffff800010127bac: ldr x1, [x19, #2440] 0.00 : ffff800010127bb0: add x0, x0, x2 : pick_next_task_fair(): : } 0.00 : ffff800010127bb4: ldr x23, [sp, #48] : update_idle_rq_clock_pelt(): 0.00 : ffff800010127bb8: sub x0, x0, x1 0.00 : ffff800010127bbc: str x0, [x19, #2448] : pick_next_task_fair(): 0.00 : ffff800010127bc0: mov x0, x21 0.00 : ffff800010127bc4: ldr x21, [sp, #32] 0.00 : ffff800010127bc8: ldp x19, x20, [sp, #16] 0.00 : ffff800010127bcc: ldp x29, x30, [sp], #96 0.00 : ffff800010127bd0: ret 0.00 : ffff800010127bd4: str x22, [x29, #40] 0.00 : ffff800010127bd8: b ffff800010127bf4 0.00 : ffff800010127bdc: mov x1, #0x0 // #0 : se = pick_next_entity(cfs_rq, curr); 0.00 : ffff800010127be0: mov x0, x20 0.00 : ffff800010127be4: bl ffff80001011ddd0 : cfs_rq = group_cfs_rq(se); 0.00 : ffff800010127be8: ldr x20, [x0, #136] : se = pick_next_entity(cfs_rq, curr); 0.00 : ffff800010127bec: mov x22, x0 : } while (cfs_rq); 0.00 : ffff800010127bf0: cbz x20, ffff800010127c18 : struct sched_entity *curr = cfs_rq->curr; 0.00 : ffff800010127bf4: ldr x1, [x20, #72] : if (curr) { 0.00 : ffff800010127bf8: cbz x1, ffff800010127be0 : if (curr->on_rq) 0.00 : ffff800010127bfc: ldr w0, [x1, #64] 0.00 : ffff800010127c00: cbz w0, ffff800010127bdc 0.00 : ffff800010127c04: str x1, [x29, #88] : update_curr(cfs_rq); 0.00 : ffff800010127c08: mov x0, x20 0.00 : ffff800010127c0c: bl ffff80001011fab8 0.00 : ffff800010127c10: ldr x1, [x29, #88] 0.00 : ffff800010127c14: b ffff800010127be0 : task_of(): : return container_of(se, struct task_struct, se); 0.00 : ffff800010127c18: sub x21, x0, #0xc0 : pick_next_task_fair(): : if (prev != p) { 0.00 : ffff800010127c1c: cmp x23, x21 0.00 : ffff800010127c20: b.eq ffff800010127c9c // b.none : struct sched_entity *pse = &prev->se; 0.00 : ffff800010127c24: add x23, x23, #0xc0 0.00 : ffff800010127c28: stp x24, x25, [x29, #56] 0.00 : ffff800010127c2c: nop : while (!(cfs_rq = is_same_group(se, pse))) { 0.00 : ffff800010127c30: ldr x20, [x22, #128] 0.00 : ffff800010127c34: ldr x0, [x23, #128] 0.00 : ffff800010127c38: cmp x20, #0x0 0.00 : ffff800010127c3c: ccmp x20, x0, #0x0, ne // ne = any 0.00 : ffff800010127c40: b.eq ffff800010127c78 // b.none : int se_depth = se->depth; 0.00 : ffff800010127c44: ldr w25, [x22, #112] : int pse_depth = pse->depth; 0.00 : ffff800010127c48: ldr w24, [x23, #112] : if (se_depth <= pse_depth) { 0.00 : ffff800010127c4c: cmp w25, w24 0.00 : ffff800010127c50: b.le ffff800010127cd0 : set_next_entity(cfs_rq_of(se), se); 0.00 : ffff800010127c54: mov x0, x20 0.00 : ffff800010127c58: mov x1, x22 0.00 : ffff800010127c5c: bl ffff80001011f9c0 : se = parent_entity(se); 0.00 : ffff800010127c60: ldr x22, [x22, #120] : while (!(cfs_rq = is_same_group(se, pse))) { 0.00 : ffff800010127c64: ldr x0, [x23, #128] 0.00 : ffff800010127c68: ldr x20, [x22, #128] 0.00 : ffff800010127c6c: cmp x20, #0x0 0.00 : ffff800010127c70: ccmp x20, x0, #0x0, ne // ne = any 0.00 : ffff800010127c74: b.ne ffff800010127c44 // b.any : put_prev_entity(cfs_rq, pse); 0.00 : ffff800010127c78: mov x1, x23 0.00 : ffff800010127c7c: mov x0, x20 0.00 : ffff800010127c80: bl ffff8000101209d0 : set_next_entity(cfs_rq, se); 0.00 : ffff800010127c84: mov x1, x22 0.00 : ffff800010127c88: mov x0, x20 0.00 : ffff800010127c8c: bl ffff80001011f9c0 0.00 : ffff800010127c90: ldr x22, [x29, #40] 0.00 : ffff800010127c94: ldp x24, x25, [x29, #56] 0.00 : ffff800010127c98: b ffff800010127aa8 0.00 : ffff800010127c9c: ldr x22, [x29, #40] 0.00 : ffff800010127ca0: b ffff800010127aa8 : task_cfs_rq(): : return p->se.cfs_rq; 0.00 : ffff800010127ca4: ldr x6, [x21, #320] : task_h_load(): : : static unsigned long task_h_load(struct task_struct *p) : { : struct cfs_rq *cfs_rq = task_cfs_rq(p); : : update_cfs_rq_h_load(cfs_rq); 0.00 : ffff800010127ca8: mov x0, x6 0.00 : ffff800010127cac: bl ffff80001011c898 : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.00 : ffff800010127cb0: ldr x2, [x6, #280] 0.00 : ffff800010127cb4: ldr x0, [x21, #416] 0.00 : ffff800010127cb8: ldr x1, [x6, #160] 0.00 : ffff800010127cbc: add x1, x1, #0x1 0.00 : ffff800010127cc0: mul x0, x0, x2 : div64_u64(): : * : * Return: dividend / divisor : */ : static inline u64 div64_u64(u64 dividend, u64 divisor) : { : return dividend / divisor; 0.00 : ffff800010127cc4: udiv x0, x0, x1 : update_misfit_status(): : rq->misfit_task_load = task_h_load(p); 0.00 : ffff800010127cc8: str x0, [x19, #2512] 0.00 : ffff800010127ccc: b ffff800010127b10 : pick_next_task_fair(): : put_prev_entity(cfs_rq_of(pse), pse); 0.00 : ffff800010127cd0: mov x1, x23 0.00 : ffff800010127cd4: bl ffff8000101209d0 : if (se_depth >= pse_depth) { 0.00 : ffff800010127cd8: cmp w25, w24 : pse = parent_entity(pse); 0.00 : ffff800010127cdc: ldr x23, [x23, #120] : if (se_depth >= pse_depth) { 0.00 : ffff800010127ce0: b.lt ffff800010127c30 // b.tstop 0.00 : ffff800010127ce4: ldr x20, [x22, #128] 0.00 : ffff800010127ce8: b ffff800010127c54 : put_prev_task(): : WARN_ON_ONCE(rq->curr != prev); 0.00 : ffff800010127cec: brk #0x800 0.00 : ffff800010127cf0: ldr x1, [x23, #128] 0.00 : ffff800010127cf4: b ffff800010127a70 Percent | Source code & Disassembly of vmlinux for cycles (203 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010702a00 : : credit_entropy_bits(): : { : int entropy_count, orig, has_initialized = 0; : const int pool_size = r->poolinfo->poolfracbits; : int nfrac = nbits << ENTROPY_SHIFT; : : if (!nbits) 0.00 : ffff800010702a00: cbz w0, ffff800010702b18 : static void credit_entropy_bits(struct entropy_store *r, int nbits) 0.00 : ffff800010702a04: stp x29, x30, [sp, #-96]! 0.00 : ffff800010702a08: mov x29, sp 3.41 : ffff800010702a0c: str x27, [sp, #80] : const int pool_size = r->poolinfo->poolfracbits; 0.00 : ffff800010702a10: adrp x27, ffff800011a87000 0.00 : ffff800010702a14: add x1, x27, #0xc18 : static void credit_entropy_bits(struct entropy_store *r, int nbits) 0.00 : ffff800010702a18: stp x19, x20, [sp, #16] 0.00 : ffff800010702a1c: stp x21, x22, [sp, #32] : int nfrac = nbits << ENTROPY_SHIFT; 0.00 : ffff800010702a20: lsl w21, w0, #3 : static void credit_entropy_bits(struct entropy_store *r, int nbits) 0.00 : ffff800010702a24: stp x23, x24, [sp, #48] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010702a28: mov x19, x1 : credit_entropy_bits(): 1.49 : ffff800010702a2c: stp x25, x26, [sp, #64] : pnfrac -= anfrac; : } while (unlikely(entropy_count < pool_size-2 && pnfrac)); : } : : if (WARN_ON(entropy_count < 0)) { : pr_warn("negative entropy/overflow: pool %s count %d\n", 0.00 : ffff800010702a30: adrp x26, ffff800011267000 : __lse__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 0.00 : ffff800010702a34: add x25, x1, #0x50 : credit_entropy_bits(): 0.00 : ffff800010702a38: add x26, x26, #0xfc8 : const int pool_size = r->poolinfo->poolfracbits; 0.00 : ffff800010702a3c: ldr x0, [x1, #48] 0.00 : ffff800010702a40: ldr w22, [x0, #12] 0.00 : ffff800010702a44: sub w24, w22, #0x2 : unsigned int anfrac = min(pnfrac, pool_size/2); 0.00 : ffff800010702a48: add w23, w22, w22, lsr #31 0.00 : ffff800010702a4c: asr w23, w23, #1 : __read_once_size(): 0.00 : ffff800010702a50: ldr w20, [x19, #80] : credit_entropy_bits(): : entropy_count += nfrac; 0.00 : ffff800010702a54: add w0, w21, w20 : if (nfrac < 0) { 0.00 : ffff800010702a58: tbnz w21, #31, ffff800010702a98 : const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2; 1.93 : ffff800010702a5c: ldr x1, [x19, #48] 0.00 : ffff800010702a60: mov w2, w21 : entropy_count = orig = READ_ONCE(r->entropy_count); 0.00 : ffff800010702a64: mov w0, w20 : const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2; 0.00 : ffff800010702a68: ldr w4, [x1] 0.00 : ffff800010702a6c: add w4, w4, #0x5 : unsigned int anfrac = min(pnfrac, pool_size/2); 0.00 : ffff800010702a70: cmp w23, w2 : ((pool_size - entropy_count)*anfrac*3) >> s; 0.00 : ffff800010702a74: sub w1, w22, w0 : unsigned int anfrac = min(pnfrac, pool_size/2); 0.00 : ffff800010702a78: csel w3, w23, w2, le : } while (unlikely(entropy_count < pool_size-2 && pnfrac)); 0.00 : ffff800010702a7c: subs w2, w2, w3 : ((pool_size - entropy_count)*anfrac*3) >> s; 0.00 : ffff800010702a80: mul w1, w1, w3 0.00 : ffff800010702a84: add w1, w1, w1, lsl #1 : unsigned int add = 0.00 : ffff800010702a88: lsr w1, w1, w4 : entropy_count += add; 0.00 : ffff800010702a8c: add w0, w1, w0 : } while (unlikely(entropy_count < pool_size-2 && pnfrac)); 1.53 : ffff800010702a90: ccmp w0, w24, #0x0, ne // ne = any 0.00 : ffff800010702a94: b.lt ffff800010702a70 // b.tstop 0.00 : ffff800010702a98: cmp w22, w0 0.00 : ffff800010702a9c: csel w3, w22, w0, le 0.00 : ffff800010702aa0: mov w2, w3 : if (WARN_ON(entropy_count < 0)) { 0.00 : ffff800010702aa4: tbnz w0, #31, ffff800010702b1c : r->name, entropy_count); : entropy_count = 0; : } else if (entropy_count > pool_size) : entropy_count = pool_size; : if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig) 1.44 : ffff800010702aa8: sxtw x1, w20 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010702aac: b ffff800010702b08 2.45 : ffff800010702ab0: b ffff800010702b08 : __lse__cmpxchg_case_mb_32(): 0.49 : ffff800010702ab4: mov x0, x25 : __cmpxchg_mb(): : } : : __CMPXCHG_GEN() : __CMPXCHG_GEN(_acq) : __CMPXCHG_GEN(_rel) : __CMPXCHG_GEN(_mb) 0.00 : ffff800010702ab8: mov w1, w20 : __lse__cmpxchg_case_mb_32(): 0.00 : ffff800010702abc: add x4, x19, #0x50 0.00 : ffff800010702ac0: mov w5, w1 0.00 : ffff800010702ac4: casal w5, w2, [x4] 80.86 : ffff800010702ac8: mov w0, w5 : credit_entropy_bits(): 0.00 : ffff800010702acc: cmp w20, w0 0.00 : ffff800010702ad0: b.ne ffff800010702a50 // b.any : entropy_count >> ENTROPY_SHIFT, _RET_IP_); : : if (r == &input_pool) { : int entropy_bits = entropy_count >> ENTROPY_SHIFT; : : if (crng_init < 2) { 0.00 : ffff800010702ad4: adrp x1, ffff800011b4c000 0.00 : ffff800010702ad8: add x0, x1, #0x520 : if (entropy_bits < 128) 0.00 : ffff800010702adc: cmp w3, #0x3ff 2.45 : ffff800010702ae0: ldr w1, [x1, #1312] 0.00 : ffff800010702ae4: ccmp w1, #0x1, #0x0, gt 0.00 : ffff800010702ae8: b.le ffff800010702b3c : return; : crng_reseed(&primary_crng, r); : entropy_bits = ENTROPY_BITS(r); : } : } : } 0.00 : ffff800010702aec: ldp x19, x20, [sp, #16] 0.00 : ffff800010702af0: ldp x21, x22, [sp, #32] 3.94 : ffff800010702af4: ldp x23, x24, [sp, #48] 0.00 : ffff800010702af8: ldp x25, x26, [sp, #64] 0.00 : ffff800010702afc: ldr x27, [sp, #80] 0.00 : ffff800010702b00: ldp x29, x30, [sp], #96 0.00 : ffff800010702b04: ret : __ll_sc__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 0.00 : ffff800010702b08: and x1, x1, #0xffffffff 0.00 : ffff800010702b0c: add x5, x19, #0x50 0.00 : ffff800010702b10: b ffff800010703f7c 0.00 : ffff800010702b14: b ffff800010702acc 0.00 : ffff800010702b18: ret : credit_entropy_bits(): : if (WARN_ON(entropy_count < 0)) { 0.00 : ffff800010702b1c: brk #0x800 : pr_warn("negative entropy/overflow: pool %s count %d\n", 0.00 : ffff800010702b20: ldr x1, [x19, #64] 0.00 : ffff800010702b24: mov w2, w0 0.00 : ffff800010702b28: mov x0, x26 0.00 : ffff800010702b2c: bl ffff80001014a85c 0.00 : ffff800010702b30: mov w2, #0x0 // #0 : entropy_count = 0; 0.00 : ffff800010702b34: mov w3, #0x0 // #0 0.00 : ffff800010702b38: b ffff800010702aa8 : crng_reseed(&primary_crng, r); 0.00 : ffff800010702b3c: add x1, x27, #0xc18 0.00 : ffff800010702b40: add x0, x0, #0x8 0.00 : ffff800010702b44: add x1, x1, #0x30 0.00 : ffff800010702b48: bl ffff800010702588 0.00 : ffff800010702b4c: b ffff800010702aec Percent | Source code & Disassembly of vmlinux for cycles (195 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001046c980 : : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 5.20 : ffff80001046c980: ldr x1, [x0, #24] : blk_mq_sched_restart(): : } : EXPORT_SYMBOL_GPL(blk_mq_sched_mark_restart_hctx); : : void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx) : { : if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) 0.00 : ffff80001046c984: tst w1, #0x4 0.00 : ffff80001046c988: b.ne ffff80001046c990 // b.any 94.80 : ffff80001046c98c: ret : { 0.00 : ffff80001046c990: stp x29, x30, [sp, #-16]! : return; : clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); 0.00 : ffff80001046c994: add x2, x0, #0x18 : { 0.00 : ffff80001046c998: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001046c99c: b ffff80001046c9bc 0.00 : ffff80001046c9a0: b ffff80001046c9bc : __lse_atomic64_andnot(): : " " #asm_op " %[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) 0.00 : ffff80001046c9a4: mov x1, #0x4 // #4 0.00 : ffff80001046c9a8: stclr x1, [x2] : blk_mq_sched_restart(): : : blk_mq_run_hw_queue(hctx, true); 0.00 : ffff80001046c9ac: mov w1, #0x1 // #1 0.00 : ffff80001046c9b0: bl ffff800010465418 : } 0.00 : ffff80001046c9b4: ldp x29, x30, [sp], #16 0.00 : ffff80001046c9b8: ret : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff80001046c9bc: mov x1, #0x4 // #4 0.00 : ffff80001046c9c0: add x4, x0, #0x18 0.00 : ffff80001046c9c4: b ffff80001046d384 : blk_mq_sched_restart(): : blk_mq_run_hw_queue(hctx, true); 0.00 : ffff80001046c9c8: mov w1, #0x1 // #1 0.00 : ffff80001046c9cc: bl ffff800010465418 : } 0.00 : ffff80001046c9d0: ldp x29, x30, [sp], #16 0.00 : ffff80001046c9d4: ret Percent | Source code & Disassembly of vmlinux for cycles (199 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010161288 : : rcu_sched_clock_irq(): : * core processing. If the current grace period has gone on too long, : * it will ask the scheduler to manufacture a context switch for the sole : * purpose of providing a providing the needed quiescent state. : */ : void rcu_sched_clock_irq(int user) : { 0.53 : ffff800010161288: stp x29, x30, [sp, #-176]! 0.00 : ffff80001016128c: mov x29, sp 0.00 : ffff800010161290: stp x19, x20, [sp, #16] : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff800010161294: mov x6, sp : rcu_sched_clock_irq(): 0.00 : ffff800010161298: stp x21, x22, [sp, #32] 0.50 : ffff80001016129c: str x23, [sp, #48] 0.00 : ffff8000101612a0: stp x26, x27, [sp, #72] : trace_rcu_utilization(TPS("Start scheduler-tick")); : raw_cpu_inc(rcu_data.ticks_this_gp); 0.00 : ffff8000101612a4: adrp x26, ffff800011529000 : { 0.00 : ffff8000101612a8: str x28, [sp, #88] : raw_cpu_inc(rcu_data.ticks_this_gp); 0.00 : ffff8000101612ac: add x3, x26, #0xac0 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000101612b0: mrs x7, tpidr_el1 : rcu_sched_clock_irq(): 0.00 : ffff8000101612b4: add x1, x3, #0x28 : /* The load-acquire pairs with the store-release setting to true. */ : if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { 0.00 : ffff8000101612b8: add x2, x3, #0xdd : raw_cpu_inc(rcu_data.ticks_this_gp); 0.00 : ffff8000101612bc: ldr x4, [x1, x7] 0.00 : ffff8000101612c0: add x4, x4, #0x1 0.53 : ffff8000101612c4: str x4, [x1, x7] : __my_cpu_offset(): 0.00 : ffff8000101612c8: mrs x1, tpidr_el1 : rcu_sched_clock_irq(): : if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { 0.00 : ffff8000101612cc: add x1, x2, x1 0.00 : ffff8000101612d0: ldarb w1, [x1] 0.00 : ffff8000101612d4: tst w1, #0xff 0.00 : ffff8000101612d8: b.eq ffff800010161308 // b.none : rcu_is_cpu_rrupt_from_idle(): : if (__this_cpu_read(rcu_data.dynticks_nmi_nesting) != 1) 0.00 : ffff8000101612dc: add x1, x3, #0xd0 : __my_cpu_offset(): : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff8000101612e0: mov x2, sp : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000101612e4: mrs x4, tpidr_el1 : rcu_is_cpu_rrupt_from_idle(): 0.00 : ffff8000101612e8: ldr x1, [x1, x4] 0.00 : ffff8000101612ec: cmp x1, #0x1 0.00 : ffff8000101612f0: b.eq ffff800010161674 // b.none : rcu_sched_clock_irq(): : /* Idle and userspace execution already are quiescent states. */ : if (!rcu_is_cpu_rrupt_from_idle() && !user) { 0.00 : ffff8000101612f4: cbz w0, ffff800010161684 : set_tsk_need_resched(current); : set_preempt_need_resched(); : } : __this_cpu_write(rcu_data.rcu_urgent_qs, false); 0.00 : ffff8000101612f8: add x1, x26, #0xac0 0.00 : ffff8000101612fc: add x1, x1, #0xdd : __my_cpu_offset(): 0.00 : ffff800010161300: mrs x2, tpidr_el1 : rcu_sched_clock_irq(): 0.00 : ffff800010161304: strb wzr, [x1, x2] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 1.02 : ffff800010161308: mrs x3, sp_el0 : rcu_flavor_sched_clock_irq(): : */ : static void rcu_flavor_sched_clock_irq(int user) : { : struct task_struct *t = current; : : if (user || rcu_is_cpu_rrupt_from_idle()) { 0.00 : ffff80001016130c: cbnz w0, ffff800010161600 : rcu_is_cpu_rrupt_from_idle(): : if (__this_cpu_read(rcu_data.dynticks_nmi_nesting) != 1) 0.00 : ffff800010161310: add x1, x26, #0xac0 0.00 : ffff800010161314: add x0, x1, #0xd0 : __my_cpu_offset(): 0.00 : ffff800010161318: mrs x2, tpidr_el1 : rcu_is_cpu_rrupt_from_idle(): 0.53 : ffff80001016131c: ldr x0, [x0, x2] 0.00 : ffff800010161320: cmp x0, #0x1 0.00 : ffff800010161324: b.eq ffff8000101615f0 // b.none : get_current(): 0.00 : ffff800010161328: mrs x4, sp_el0 : rcu_flavor_sched_clock_irq(): : rcu_note_voluntary_context_switch(current); : } : if (rcu_preempt_depth() > 0 || 0.00 : ffff80001016132c: ldr w0, [x4, #776] 0.00 : ffff800010161330: cmp w0, #0x0 0.00 : ffff800010161334: b.gt ffff800010161344 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010161338: ldr w0, [x4, #16] : rcu_flavor_sched_clock_irq(): 0.00 : ffff80001016133c: and w0, w0, #0xffff 0.00 : ffff800010161340: cbz w0, ffff80001016164c : (preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK))) { : /* No QS, force context switch if deferred. */ : if (rcu_preempt_need_deferred_qs(t)) { 0.00 : ffff800010161344: mov x0, x3 0.00 : ffff800010161348: bl ffff80001015df30 0.00 : ffff80001016134c: tst w0, #0xff 0.00 : ffff800010161350: b.ne ffff800010161628 // b.any : get_current(): 0.00 : ffff800010161354: mrs x0, sp_el0 : rcu_flavor_sched_clock_irq(): : rcu_qs(); /* Report immediate QS. */ : return; : } : : /* If GP is oldish, ask for help from rcu_read_unlock_special(). */ : if (rcu_preempt_depth() > 0 && 0.51 : ffff800010161358: ldr w0, [x0, #776] 0.00 : ffff80001016135c: mov x2, sp 0.00 : ffff800010161360: adrp x20, ffff800011922000 0.00 : ffff800010161364: cmp w0, #0x0 0.00 : ffff800010161368: b.le ffff8000101613b0 : __this_cpu_read(rcu_data.core_needs_qs) && 0.00 : ffff80001016136c: add x1, x26, #0xac0 0.00 : ffff800010161370: add x0, x1, #0x12 : __my_cpu_offset(): 0.00 : ffff800010161374: mrs x4, tpidr_el1 : rcu_flavor_sched_clock_irq(): : if (rcu_preempt_depth() > 0 && 0.00 : ffff800010161378: ldrb w0, [x0, x4] 0.00 : ffff80001016137c: cbz w0, ffff8000101613b0 : __this_cpu_read(rcu_data.cpu_no_qs.b.norm) && 0.00 : ffff800010161380: add x1, x1, #0x10 : __this_cpu_read(rcu_data.core_needs_qs) && 0.00 : ffff800010161384: ldrb w0, [x1, x4] 0.00 : ffff800010161388: cbz w0, ffff8000101613b0 : __this_cpu_read(rcu_data.cpu_no_qs.b.norm) && 0.00 : ffff80001016138c: ldrb w0, [x3, #781] 0.00 : ffff800010161390: cbnz w0, ffff8000101613b0 : !t->rcu_read_unlock_special.b.need_qs && : time_after(jiffies, rcu_state.gp_start + HZ)) 0.00 : ffff800010161394: adrp x0, ffff800011907000 0.00 : ffff800010161398: add x1, x20, #0xc80 0.00 : ffff80001016139c: ldr x4, [x0, #2432] 0.53 : ffff8000101613a0: ldr x0, [x1, #9056] 0.00 : ffff8000101613a4: sub x0, x0, x4 0.00 : ffff8000101613a8: cmn x0, #0xfa 0.00 : ffff8000101613ac: b.mi ffff8000101616c4 // b.first : check_cpu_stall(): : unsigned long j; : unsigned long jn; : unsigned long js; : struct rcu_node *rnp; : : if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) || 1.04 : ffff8000101613b0: adrp x23, ffff80001190a000 <__per_cpu_offset+0x6d8> : rcu_pending(): : * stalls first, else we might not get a chance. : */ : static int rcu_pending(int user) : { : bool gp_in_progress; : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff8000101613b4: add x19, x26, #0xac0 : __my_cpu_offset(): 0.00 : ffff8000101613b8: mrs x27, tpidr_el1 : check_cpu_stall(): 0.00 : ffff8000101613bc: ldr w0, [x23, #656] : rcu_pending(): 0.00 : ffff8000101613c0: add x28, x19, x27 : struct rcu_node *rnp = rdp->mynode; 0.00 : ffff8000101613c4: ldr x21, [x28, #24] : check_cpu_stall(): 0.00 : ffff8000101613c8: cbz w0, ffff8000101613f8 0.00 : ffff8000101613cc: adrp x22, ffff800011b20000 <__log_buf+0x1fc10> 0.00 : ffff8000101613d0: add x0, x22, #0xd78 0.00 : ffff8000101613d4: ldrb w0, [x0, #64] 0.00 : ffff8000101613d8: cbz w0, ffff800010161540 : __read_once_size(): 0.00 : ffff8000101613dc: add x0, x20, #0xc80 0.00 : ffff8000101613e0: ldr x0, [x0, #8776] : check_cpu_stall(): 0.00 : ffff8000101613e4: tst x0, #0x3 0.00 : ffff8000101613e8: b.eq ffff800010161540 // b.none 0.00 : ffff8000101613ec: stp x24, x25, [x29, #56] : rcu_stall_kick_kthreads(): 0.00 : ffff8000101613f0: bl ffff80001015e080 0.00 : ffff8000101613f4: b ffff80001016141c : __read_once_size(): 0.00 : ffff8000101613f8: add x0, x20, #0xc80 0.49 : ffff8000101613fc: ldr x0, [x0, #8776] : check_cpu_stall(): 0.00 : ffff800010161400: tst x0, #0x3 0.00 : ffff800010161404: b.eq ffff800010161540 // b.none : rcu_stall_kick_kthreads(): : if (!rcu_kick_kthreads) 13.79 : ffff800010161408: adrp x22, ffff800011b20000 <__log_buf+0x1fc10> 0.00 : ffff80001016140c: add x0, x22, #0xd78 0.00 : ffff800010161410: stp x24, x25, [x29, #56] 0.00 : ffff800010161414: ldrb w0, [x0, #64] 0.00 : ffff800010161418: cbnz w0, ffff8000101613f0 : __read_once_size(): 1.52 : ffff80001016141c: add x0, x20, #0xc80 : check_cpu_stall(): : !rcu_gp_in_progress()) : return; : rcu_stall_kick_kthreads(); : j = jiffies; 0.00 : ffff800010161420: adrp x24, ffff800011907000 0.00 : ffff800010161424: ldr x2, [x24, #2432] : __read_once_size(): 5.50 : ffff800010161428: ldr x25, [x0, #8776] : check_cpu_stall(): : * : * Given this check, comparisons of jiffies, rcu_state.jiffies_stall, : * and rcu_state.gp_start suffice to forestall false positives. : */ : gs1 = READ_ONCE(rcu_state.gp_seq); : smp_rmb(); /* Pick up ->gp_seq first... */ 0.00 : ffff80001016142c: dmb ishld : __read_once_size(): 47.95 : ffff800010161430: ldr x3, [x0, #9088] : check_cpu_stall(): : js = READ_ONCE(rcu_state.jiffies_stall); : smp_rmb(); /* ...then ->jiffies_stall before the rest... */ 1.56 : ffff800010161434: dmb ishld : __read_once_size(): 4.06 : ffff800010161438: ldr x1, [x0, #9056] : check_cpu_stall(): : gps = READ_ONCE(rcu_state.gp_start); : smp_rmb(); /* ...and finally ->gp_start before ->gp_seq again. */ 5.09 : ffff80001016143c: dmb ishld : __read_once_size(): 3.55 : ffff800010161440: ldr x0, [x0, #8776] : check_cpu_stall(): : gs2 = READ_ONCE(rcu_state.gp_seq); : if (gs1 != gs2 || 0.00 : ffff800010161444: cmp x25, x0 0.00 : ffff800010161448: b.ne ffff800010161620 // b.any 0.00 : ffff80001016144c: subs x4, x2, x3 0.00 : ffff800010161450: b.mi ffff800010161620 // b.first : ULONG_CMP_LT(j, js) || : ULONG_CMP_GE(gps, js)) 0.00 : ffff800010161454: sub x1, x1, x3 : ULONG_CMP_LT(j, js) || 0.00 : ffff800010161458: tbz x1, #63, ffff800010161620 : __read_once_size(): 0.00 : ffff80001016145c: adrp x12, ffff80001190a000 <__per_cpu_offset+0x6d8> : check_cpu_stall(): : return; /* No stall or GP completed since entering function. */ : rnp = rdp->mynode; 0.00 : ffff800010161460: ldr x2, [x28, #24] : __read_once_size(): 0.00 : ffff800010161464: ldr w0, [x12, #660] : rcu_jiffies_till_stall_check(): : if (till_stall_check < 3) { 0.00 : ffff800010161468: cmp w0, #0x2 0.00 : ffff80001016146c: b.gt ffff8000101616d0 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010161470: mov w1, #0x3 // #3 0.00 : ffff800010161474: mov x0, #0x8ca // #2250 0.00 : ffff800010161478: str w1, [x12, #660] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001016147c: add x1, x20, #0xc80 : check_cpu_stall(): : jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3; 0.00 : ffff800010161480: ldr x13, [x24, #2432] 0.00 : ffff800010161484: add x13, x13, #0x3 : __read_once_size(): 0.00 : ffff800010161488: ldr x1, [x1, #8776] : check_cpu_stall(): 0.00 : ffff80001016148c: add x13, x13, x0 : if (rcu_gp_in_progress() && 0.00 : ffff800010161490: tst x1, #0x3 0.00 : ffff800010161494: b.eq ffff8000101614a8 // b.none : __read_once_size(): 0.00 : ffff800010161498: ldr x1, [x2, #32] : check_cpu_stall(): : (READ_ONCE(rnp->qsmask) & rdp->grpmask) && 0.00 : ffff80001016149c: ldr x0, [x28, #32] : if (rcu_gp_in_progress() && 0.00 : ffff8000101614a0: tst x1, x0 0.00 : ffff8000101614a4: b.ne ffff800010161714 // b.any : __read_once_size(): 0.00 : ffff8000101614a8: add x0, x20, #0xc80 0.00 : ffff8000101614ac: ldr x0, [x0, #8776] : check_cpu_stall(): : /* We haven't checked in, so go dump stack. */ : print_cpu_stall(); : if (rcu_cpu_stall_ftrace_dump) : rcu_ftrace_dump(DUMP_ALL); : : } else if (rcu_gp_in_progress() && 0.00 : ffff8000101614b0: tst x0, #0x3 0.00 : ffff8000101614b4: b.eq ffff800010161620 // b.none 0.00 : ffff8000101614b8: cmp x4, #0x2 0.00 : ffff8000101614bc: b.mi ffff800010161620 // b.first : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101614c0: b ffff8000101616e8 0.00 : ffff8000101614c4: b ffff8000101616e8 : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff8000101614c8: add x4, x20, #0xc80 0.00 : ffff8000101614cc: mov x5, #0x2380 // #9088 0.00 : ffff8000101614d0: adrp x0, ffff800011925000 0.00 : ffff8000101614d4: mov x1, x3 0.00 : ffff8000101614d8: add x0, x0, #0x0 0.00 : ffff8000101614dc: mov x2, x13 0.00 : ffff8000101614e0: add x4, x4, x5 0.00 : ffff8000101614e4: mov x5, x1 0.00 : ffff8000101614e8: casal x5, x2, [x4] 0.00 : ffff8000101614ec: mov x0, x5 : check_cpu_stall(): : ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) && 0.00 : ffff8000101614f0: cmp x3, x0 0.00 : ffff8000101614f4: b.ne ffff800010161620 // b.any : rcu_stall_kick_kthreads(): : if (!rcu_kick_kthreads) 0.00 : ffff8000101614f8: add x0, x22, #0xd78 0.00 : ffff8000101614fc: ldrb w0, [x0, #64] 0.00 : ffff800010161500: cbnz w0, ffff8000101617c0 : print_other_cpu_stall(): : if (rcu_cpu_stall_suppress) 0.00 : ffff800010161504: ldr w0, [x23, #656] 0.00 : ffff800010161508: cbz w0, ffff80001016187c : check_cpu_stall(): : cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) { : : /* They had a few time units to dump stack, so complain. */ : print_other_cpu_stall(gs2); : if (rcu_cpu_stall_ftrace_dump) 0.00 : ffff80001016150c: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010161510: ldr w0, [x0, #664] 0.00 : ffff800010161514: cbz w0, ffff800010161620 : __read_once_size(): 0.00 : ffff800010161518: add x0, x22, #0xd78 0.00 : ffff80001016151c: ldr w1, [x0, #80] : check_cpu_stall(): : rcu_ftrace_dump(DUMP_ALL); 0.00 : ffff800010161520: cbnz w1, ffff800010161620 : __xchg_mb(): : } : : __XCHG_GEN() : __XCHG_GEN(_acq) : __XCHG_GEN(_rel) : __XCHG_GEN(_mb) 0.00 : ffff800010161524: add x0, x0, #0x50 0.00 : ffff800010161528: bl ffff80001015d4a8 <__xchg_case_mb_32.constprop.96> : check_cpu_stall(): 0.00 : ffff80001016152c: cbnz w0, ffff800010161620 : rcu_ftrace_dump(DUMP_ALL); 0.00 : ffff800010161530: ldr w0, [x23, #656] 0.00 : ffff800010161534: cbnz w0, ffff800010161618 0.00 : ffff800010161538: ldp x24, x25, [x29, #56] 0.00 : ffff80001016153c: str wzr, [x23, #656] : __read_once_size(): 0.50 : ffff800010161540: add x20, x20, #0xc80 : rcu_pending(): : if ((user || rcu_is_cpu_rrupt_from_idle()) && rcu_nohz_full_cpu()) : return 0; : : /* Is the RCU core waiting for a quiescent state from this CPU? */ : gp_in_progress = rcu_gp_in_progress(); : if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress) 0.00 : ffff800010161544: ldrb w1, [x28, #18] : __read_once_size(): 0.00 : ffff800010161548: ldr x0, [x20, #8776] : rcu_seq_state(): : * Return the state portion of a sequence number previously returned : * by rcu_seq_snap() or rcu_seq_current(). : */ : static inline int rcu_seq_state(unsigned long s) : { : return s & RCU_SEQ_STATE_MASK; 0.00 : ffff80001016154c: and w20, w0, #0x3 : rcu_pending(): 0.00 : ffff800010161550: cbz w1, ffff8000101615a4 0.00 : ffff800010161554: ldrb w0, [x28, #16] 0.00 : ffff800010161558: cbnz w0, ffff8000101615a4 0.00 : ffff80001016155c: cbnz w20, ffff800010161584 : return 1; : : /* Does this CPU have callbacks ready to invoke? */ : if (rcu_segcblist_ready_cbs(&rdp->cblist)) 0.00 : ffff800010161560: add x0, x28, #0x50 0.00 : ffff800010161564: bl ffff800010162b90 0.00 : ffff800010161568: tst w0, #0xff 0.00 : ffff80001016156c: b.ne ffff800010161584 // b.any : return 1; : : /* Has RCU gone idle with this CPU needing another grace period? */ : if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && 0.00 : ffff800010161570: ldrb w0, [x28, #160] 0.00 : ffff800010161574: cbz w0, ffff8000101615b8 : __read_once_size(): 0.49 : ffff800010161578: ldr x0, [x28, #104] 0.00 : ffff80001016157c: ldr x0, [x0] : rcu_pending(): : (!IS_ENABLED(CONFIG_RCU_NOCB_CPU) || : !rcu_segcblist_is_offloaded(&rdp->cblist)) && 0.00 : ffff800010161580: cbz x0, ffff8000101615b8 : rcu_sched_clock_irq(): : invoke_rcu_core(); 0.49 : ffff800010161584: bl ffff80001015eda8 : } 0.00 : ffff800010161588: ldr x23, [sp, #48] 0.00 : ffff80001016158c: ldp x19, x20, [sp, #16] 0.00 : ffff800010161590: ldp x21, x22, [sp, #32] 0.00 : ffff800010161594: ldp x26, x27, [sp, #72] 0.00 : ffff800010161598: ldr x28, [sp, #88] 0.00 : ffff80001016159c: ldp x29, x30, [sp], #176 0.00 : ffff8000101615a0: ret : rcu_pending(): : if (rcu_segcblist_ready_cbs(&rdp->cblist)) 0.00 : ffff8000101615a4: add x0, x28, #0x50 0.00 : ffff8000101615a8: bl ffff800010162b90 0.00 : ffff8000101615ac: tst w0, #0xff 0.00 : ffff8000101615b0: b.ne ffff800010161584 // b.any : if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && 0.00 : ffff8000101615b4: cbz w20, ffff800010161570 : __read_once_size(): 0.51 : ffff8000101615b8: ldr x1, [x21, #8] : rcu_pending(): : !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) : return 1; : : /* Have RCU grace period completed or started? */ : if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq || 8.83 : ffff8000101615bc: ldr x0, [x19, x27] 0.00 : ffff8000101615c0: cmp x1, x0 0.00 : ffff8000101615c4: b.ne ffff800010161584 // b.any : __read_once_size(): 0.00 : ffff8000101615c8: ldrb w0, [x28, #20] : rcu_pending(): 0.00 : ffff8000101615cc: tst w0, #0xff 0.00 : ffff8000101615d0: b.ne ffff800010161584 // b.any : rcu_sched_clock_irq(): : } 0.00 : ffff8000101615d4: ldp x19, x20, [sp, #16] 0.00 : ffff8000101615d8: ldp x21, x22, [sp, #32] 0.00 : ffff8000101615dc: ldr x23, [sp, #48] 0.50 : ffff8000101615e0: ldp x26, x27, [sp, #72] 0.00 : ffff8000101615e4: ldr x28, [sp, #88] 0.00 : ffff8000101615e8: ldp x29, x30, [sp], #176 0.00 : ffff8000101615ec: ret : rcu_is_cpu_rrupt_from_idle(): : return __this_cpu_read(rcu_data.dynticks_nesting) == 0; 0.00 : ffff8000101615f0: add x1, x1, #0xc8 : rcu_flavor_sched_clock_irq(): : if (user || rcu_is_cpu_rrupt_from_idle()) { 0.00 : ffff8000101615f4: ldr x0, [x1, x2] 0.00 : ffff8000101615f8: cbnz x0, ffff800010161328 0.00 : ffff8000101615fc: nop : get_current(): 0.00 : ffff800010161600: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff800010161604: ldrb w1, [x0, #816] : rcu_flavor_sched_clock_irq(): : rcu_note_voluntary_context_switch(current); 0.00 : ffff800010161608: tst w1, #0xff 0.00 : ffff80001016160c: b.eq ffff800010161328 // b.none : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff800010161610: strb wzr, [x0, #816] 0.00 : ffff800010161614: b ffff800010161328 : check_cpu_stall(): 0.00 : ffff800010161618: cmp w0, #0x3 0.00 : ffff80001016161c: b.eq ffff800010161538 // b.none 0.00 : ffff800010161620: ldp x24, x25, [x29, #56] 0.00 : ffff800010161624: b ffff800010161540 : arch_static_branch_jump(): 0.00 : ffff800010161628: b ffff80001016163c 0.00 : ffff80001016162c: b ffff80001016163c : __lse_atomic64_or(): : ATOMIC64_OP(or, stset) 0.00 : ffff800010161630: mov x0, #0x2 // #2 0.00 : ffff800010161634: stset x0, [x3] 0.00 : ffff800010161638: b ffff800010161640 : __ll_sc_atomic64_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff80001016163c: b ffff80001016265c : get_current(): 0.00 : ffff800010161640: mrs x0, sp_el0 : set_preempt_need_resched(): : task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \ : } while (0) : : static inline void set_preempt_need_resched(void) : { : current_thread_info()->preempt.need_resched = 0; 0.00 : ffff800010161644: str wzr, [x0, #20] 0.00 : ffff800010161648: b ffff800010161354 : rcu_flavor_sched_clock_irq(): : } else if (rcu_preempt_need_deferred_qs(t)) { 0.00 : ffff80001016164c: mov x0, x3 0.00 : ffff800010161650: bl ffff80001015df30 0.00 : ffff800010161654: tst w0, #0xff 0.00 : ffff800010161658: b.ne ffff8000101616b0 // b.any : } else if (!rcu_preempt_depth()) { 0.00 : ffff80001016165c: ldr w0, [x4, #776] 0.00 : ffff800010161660: cbnz w0, ffff800010161354 : rcu_qs(); /* Report immediate QS. */ 0.00 : ffff800010161664: bl ffff80001015df08 0.00 : ffff800010161668: adrp x20, ffff800011922000 0.00 : ffff80001016166c: mov x2, sp 0.00 : ffff800010161670: b ffff8000101613b0 : rcu_is_cpu_rrupt_from_idle(): 0.00 : ffff800010161674: add x3, x3, #0xc8 : rcu_sched_clock_irq(): : if (!rcu_is_cpu_rrupt_from_idle() && !user) { 0.00 : ffff800010161678: ldr x1, [x3, x4] 0.00 : ffff80001016167c: cbz x1, ffff8000101612f8 0.00 : ffff800010161680: cbnz w0, ffff8000101612f8 : get_current(): 0.00 : ffff800010161684: mrs x1, sp_el0 : arch_static_branch_jump(): 0.00 : ffff800010161688: b ffff8000101616a8 0.00 : ffff80001016168c: b ffff8000101616a8 : __lse_atomic64_or(): 0.00 : ffff800010161690: mov x2, #0x2 // #2 0.00 : ffff800010161694: stset x2, [x1] : get_current(): 0.00 : ffff800010161698: mrs x1, sp_el0 0.00 : ffff80001016169c: mov x2, sp : set_preempt_need_resched(): 0.00 : ffff8000101616a0: str wzr, [x1, #20] 0.00 : ffff8000101616a4: b ffff8000101612f8 : __ll_sc_atomic64_or(): 0.00 : ffff8000101616a8: b ffff800010162674 0.00 : ffff8000101616ac: b ffff800010161698 : rcu_flavor_sched_clock_irq(): : rcu_preempt_deferred_qs(t); /* Report deferred QS. */ 0.00 : ffff8000101616b0: mov x0, x3 0.00 : ffff8000101616b4: adrp x20, ffff800011922000 0.00 : ffff8000101616b8: bl ffff800010160210 0.00 : ffff8000101616bc: mov x2, sp 0.00 : ffff8000101616c0: b ffff8000101613b0 : t->rcu_read_unlock_special.b.need_qs = true; 0.00 : ffff8000101616c4: mov w0, #0x1 // #1 0.00 : ffff8000101616c8: strb w0, [x3, #781] 0.00 : ffff8000101616cc: b ffff8000101613b0 : rcu_jiffies_till_stall_check(): : } else if (till_stall_check > 300) { 0.00 : ffff8000101616d0: cmp w0, #0x12c 0.00 : ffff8000101616d4: b.gt ffff800010161700 0.00 : ffff8000101616d8: mov w1, #0x2ee // #750 0.00 : ffff8000101616dc: mul w0, w0, w1 0.00 : ffff8000101616e0: sxtw x0, w0 0.00 : ffff8000101616e4: b ffff80001016147c : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff8000101616e8: add x0, x20, #0xc80 0.00 : ffff8000101616ec: mov x1, #0x2380 // #9088 0.00 : ffff8000101616f0: add x0, x0, x1 0.00 : ffff8000101616f4: b ffff80001016268c 0.00 : ffff8000101616f8: mov x0, x1 0.00 : ffff8000101616fc: b ffff8000101614f0 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010161700: mov x0, #0x6ee8 // #28392 0.00 : ffff800010161704: mov w1, #0x12c // #300 0.00 : ffff800010161708: movk x0, #0x3, lsl #16 0.00 : ffff80001016170c: str w1, [x12, #660] 0.00 : ffff800010161710: b ffff80001016147c : arch_static_branch_jump(): 0.00 : ffff800010161714: b ffff8000101617a8 0.00 : ffff800010161718: b ffff8000101617a8 : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff80001016171c: add x14, x20, #0xc80 0.00 : ffff800010161720: mov x6, #0x2380 // #9088 0.00 : ffff800010161724: adrp x0, ffff800011925000 0.00 : ffff800010161728: mov x1, x3 0.00 : ffff80001016172c: add x0, x0, #0x0 0.00 : ffff800010161730: mov x2, x13 0.00 : ffff800010161734: add x5, x14, x6 0.00 : ffff800010161738: mov x6, x1 0.00 : ffff80001016173c: casal x6, x2, [x5] 0.00 : ffff800010161740: mov x0, x6 : check_cpu_stall(): : (READ_ONCE(rnp->qsmask) & rdp->grpmask) && 0.00 : ffff800010161744: cmp x3, x0 0.00 : ffff800010161748: b.ne ffff8000101614a8 // b.any : print_cpu_stall(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff80001016174c: add x0, x26, #0xac0 0.00 : ffff800010161750: mov x25, x0 0.00 : ffff800010161754: bl ffff80001015c060 <__my_cpu_offset> 0.00 : ffff800010161758: mov x2, x0 : rcu_stall_kick_kthreads(): : if (!rcu_kick_kthreads) 0.00 : ffff80001016175c: add x1, x22, #0xd78 0.00 : ffff800010161760: ldrb w0, [x1, #64] 0.00 : ffff800010161764: cbz w0, ffff800010161774 0.00 : ffff800010161768: str x2, [x29, #168] 0.00 : ffff80001016176c: bl ffff80001015e080 0.00 : ffff800010161770: ldr x2, [x29, #168] : print_cpu_stall(): : if (rcu_cpu_stall_suppress) 0.00 : ffff800010161774: ldr w0, [x23, #656] 0.00 : ffff800010161778: cbz w0, ffff8000101617c8 : check_cpu_stall(): : if (rcu_cpu_stall_ftrace_dump) 0.00 : ffff80001016177c: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010161780: ldr w0, [x0, #664] 0.00 : ffff800010161784: cbz w0, ffff800010161620 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010161788: add x0, x22, #0xd78 0.00 : ffff80001016178c: ldr w1, [x0, #72] : check_cpu_stall(): : rcu_ftrace_dump(DUMP_ALL); 0.00 : ffff800010161790: cbnz w1, ffff800010161620 : __xchg_mb(): 0.00 : ffff800010161794: add x0, x0, #0x48 0.00 : ffff800010161798: bl ffff80001015d4a8 <__xchg_case_mb_32.constprop.96> : check_cpu_stall(): 0.00 : ffff80001016179c: cbz w0, ffff800010161530 0.00 : ffff8000101617a0: ldp x24, x25, [x29, #56] 0.00 : ffff8000101617a4: b ffff800010161540 : __ll_sc__cmpxchg_case_mb_64(): 0.00 : ffff8000101617a8: add x0, x20, #0xc80 0.00 : ffff8000101617ac: mov x2, #0x2380 // #9088 0.00 : ffff8000101617b0: add x0, x0, x2 0.00 : ffff8000101617b4: b ffff8000101626ac 0.00 : ffff8000101617b8: mov x0, x1 0.00 : ffff8000101617bc: b ffff800010161744 : rcu_stall_kick_kthreads(): 0.00 : ffff8000101617c0: bl ffff80001015e080 0.00 : ffff8000101617c4: b ffff800010161504 : print_cpu_stall(): : pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name); 0.00 : ffff8000101617c8: add x1, x20, #0xc80 : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff8000101617cc: add x0, x25, x2 0.00 : ffff8000101617d0: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> : for_each_possible_cpu(cpu) 0.00 : ffff8000101617d4: add x2, x2, #0x1a0 : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff8000101617d8: stp x2, x0, [x29, #160] : rcu_get_n_cbs_cpu(): : struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); 0.00 : ffff8000101617dc: add x2, x26, #0xac0 : print_cpu_stall(): : pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name); 0.00 : ffff8000101617e0: ldr x1, [x1, #9120] 0.00 : ffff8000101617e4: adrp x0, ffff8000111c7000 : rcu_get_n_cbs_cpu(): 0.00 : ffff8000101617e8: str x2, [x29, #152] : print_cpu_stall(): 0.00 : ffff8000101617ec: add x0, x0, #0xad0 : long totqlen = 0; 0.00 : ffff8000101617f0: mov x25, #0x0 // #0 : pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name); 0.00 : ffff8000101617f4: bl ffff80001014a85c : raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags); 0.00 : ffff8000101617f8: ldr x0, [x29, #168] 0.00 : ffff8000101617fc: ldr x0, [x0, #24] 0.00 : ffff800010161800: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff800010161804: mov x26, x0 : print_cpu_stall_info(smp_processor_id()); 0.00 : ffff800010161808: adrp x2, ffff80001151d000 0.00 : ffff80001016180c: add x2, x2, #0x18 0.00 : ffff800010161810: bl ffff80001015c060 <__my_cpu_offset> 0.00 : ffff800010161814: ldr w0, [x2, x0] 0.00 : ffff800010161818: bl ffff8000101628ac : raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags); 0.00 : ffff80001016181c: ldr x0, [x29, #168] 0.00 : ffff800010161820: mov x1, x26 : rcu_get_n_cbs_cpu(): 0.00 : ffff800010161824: adrp x26, ffff800011909000 0.00 : ffff800010161828: add x26, x26, #0x928 : print_cpu_stall(): 0.00 : ffff80001016182c: ldr x0, [x0, #24] 0.00 : ffff800010161830: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010161834: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : for_each_possible_cpu(cpu) 0.00 : ffff800010161838: add x3, x1, #0x2f0 0.00 : ffff80001016183c: mov w0, #0xffffffff // #-1 0.00 : ffff800010161840: ldr x1, [x29, #160] 0.00 : ffff800010161844: str x3, [x29, #168] 0.00 : ffff800010161848: bl ffff800010cb92d8 0.00 : ffff80001016184c: ldr x3, [x29, #168] 0.00 : ffff800010161850: ldr w1, [x3] 0.00 : ffff800010161854: cmp w0, w1 0.00 : ffff800010161858: b.cs ffff800010161948 // b.hs, b.nlast : rcu_get_n_cbs_cpu(): 0.00 : ffff80001016185c: ldr x2, [x26, w0, sxtw #3] 0.00 : ffff800010161860: ldr x1, [x29, #152] 0.00 : ffff800010161864: add x1, x1, x2 : if (rcu_segcblist_is_enabled(&rdp->cblist)) 0.00 : ffff800010161868: ldrb w2, [x1, #160] 0.00 : ffff80001016186c: cbz w2, ffff800010161940 : __read_once_size(): 0.00 : ffff800010161870: ldr x1, [x1, #152] : print_cpu_stall(): : totqlen += rcu_get_n_cbs_cpu(cpu); 0.00 : ffff800010161874: add x25, x25, x1 0.00 : ffff800010161878: b ffff800010161840 : print_other_cpu_stall(): : pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name); 0.00 : ffff80001016187c: add x1, x20, #0xc80 0.00 : ffff800010161880: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010161884: str x2, [x29, #152] 0.00 : ffff800010161888: mov x2, x1 0.00 : ffff80001016188c: adrp x0, ffff8000111c7000 0.00 : ffff800010161890: str x2, [x29, #160] 0.00 : ffff800010161894: ldr x1, [x1, #9120] 0.00 : ffff800010161898: add x0, x0, #0xb20 : int ndetected = 0; 0.00 : ffff80001016189c: str wzr, [x29, #168] : pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name); 0.00 : ffff8000101618a0: bl ffff80001014a85c : rcu_for_each_leaf_node(rnp) { 0.00 : ffff8000101618a4: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101618a8: add x0, x0, #0x2a0 : rcu_print_task_stall(): : pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):", 0.00 : ffff8000101618ac: adrp x3, ffff8000111c7000 0.00 : ffff8000101618b0: add x2, x3, #0xb50 : pr_cont(" P%d", t->pid); 0.00 : ffff8000101618b4: adrp x1, ffff8000111c7000 0.00 : ffff8000101618b8: add x1, x1, #0x8f8 : print_other_cpu_stall(): : rcu_for_each_leaf_node(rnp) { 0.00 : ffff8000101618bc: ldr w0, [x0, #4] : rcu_print_task_stall(): : pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):", 0.00 : ffff8000101618c0: stp x1, x2, [x29, #128] : print_other_cpu_stall(): : rcu_for_each_leaf_node(rnp) { 0.00 : ffff8000101618c4: ldr x2, [x29, #160] 0.00 : ffff8000101618c8: sub w0, w0, #0x1 0.00 : ffff8000101618cc: add x0, x2, w0, sxtw #3 0.00 : ffff8000101618d0: ldr x4, [x0, #8704] 0.00 : ffff8000101618d4: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101618d8: add x1, x20, #0xc80 0.00 : ffff8000101618dc: ldrsw x0, [x0, #672] 0.00 : ffff8000101618e0: add x0, x1, x0, lsl #9 0.00 : ffff8000101618e4: cmp x4, x0 0.00 : ffff8000101618e8: b.cs ffff800010161a48 // b.hs, b.nlast : raw_spin_lock_irqsave_rcu_node(rnp, flags); 0.00 : ffff8000101618ec: mov x0, x4 0.00 : ffff8000101618f0: str x4, [x29, #160] 0.00 : ffff8000101618f4: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff8000101618f8: str x0, [x29, #144] : __read_once_size(): 0.00 : ffff8000101618fc: ldr x4, [x29, #160] : rcu_print_task_stall(): : return 0; 0.00 : ffff800010161900: str wzr, [x29, #160] : __read_once_size(): 0.00 : ffff800010161904: ldr x0, [x4, #144] : rcu_print_task_stall(): : if (!rcu_preempt_blocked_readers_cgp(rnp)) 0.00 : ffff800010161908: cbnz x0, ffff8000101619f8 : print_other_cpu_stall(): : ndetected += rcu_print_task_stall(rnp); 0.00 : ffff80001016190c: ldr w1, [x29, #168] 0.00 : ffff800010161910: ldr w2, [x29, #160] : if (rnp->qsmask != 0) { 0.00 : ffff800010161914: ldr x0, [x4, #32] : ndetected += rcu_print_task_stall(rnp); 0.00 : ffff800010161918: add w1, w1, w2 0.00 : ffff80001016191c: str w1, [x29, #168] : if (rnp->qsmask != 0) { 0.00 : ffff800010161920: cbnz x0, ffff800010161c98 : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff800010161924: ldr x1, [x29, #144] 0.00 : ffff800010161928: mov x0, x4 : rcu_for_each_leaf_node(rnp) { 0.00 : ffff80001016192c: add x4, x4, #0x200 0.00 : ffff800010161930: str x4, [x29, #160] : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff800010161934: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010161938: ldr x4, [x29, #160] 0.00 : ffff80001016193c: b ffff8000101618d4 : rcu_get_n_cbs_cpu(): : return 0; 0.00 : ffff800010161940: mov x1, #0x0 // #0 0.00 : ffff800010161944: b ffff800010161874 : __read_once_size(): 0.00 : ffff800010161948: add x26, x20, #0xc80 : print_cpu_stall(): : pr_cont("\t(t=%lu jiffies g=%ld q=%lu)\n", 0.00 : ffff80001016194c: ldr x4, [x24, #2432] 0.00 : ffff800010161950: mov x3, x25 0.00 : ffff800010161954: adrp x0, ffff8000111c7000 0.00 : ffff800010161958: add x0, x0, #0xb00 : __read_once_size(): 0.00 : ffff80001016195c: ldr x2, [x26, #8776] : print_cpu_stall(): 0.00 : ffff800010161960: ldr x1, [x26, #9056] 0.00 : ffff800010161964: sub x1, x4, x1 0.00 : ffff800010161968: bl ffff80001014a85c : rcu_check_gp_kthread_starvation(); 0.00 : ffff80001016196c: bl ffff8000101626fc : rcu_dump_cpu_stacks(); 0.00 : ffff800010161970: bl ffff8000101627cc : raw_spin_lock_irqsave_rcu_node(rnp, flags); 0.00 : ffff800010161974: mov x0, x26 0.00 : ffff800010161978: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall))) 0.00 : ffff80001016197c: ldr x2, [x24, #2432] : __read_once_size(): 0.00 : ffff800010161980: ldr x1, [x26, #9088] : print_cpu_stall(): 0.00 : ffff800010161984: sub x2, x2, x1 0.00 : ffff800010161988: tbnz x2, #63, ffff8000101619c0 : __read_once_size(): 0.00 : ffff80001016198c: adrp x12, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010161990: ldr w2, [x12, #660] : rcu_jiffies_till_stall_check(): : if (till_stall_check < 3) { 0.00 : ffff800010161994: cmp w2, #0x2 0.00 : ffff800010161998: b.gt ffff800010161d1c : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001016199c: mov w2, #0x3 // #3 0.00 : ffff8000101619a0: str w2, [x12, #660] : rcu_jiffies_till_stall_check(): : return till_stall_check * HZ + RCU_STALL_DELAY_DELTA; 0.00 : ffff8000101619a4: mov w1, #0x2ee // #750 : print_cpu_stall(): : WRITE_ONCE(rcu_state.jiffies_stall, 0.00 : ffff8000101619a8: ldr x4, [x24, #2432] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101619ac: add x3, x20, #0xc80 : print_cpu_stall(): 0.00 : ffff8000101619b0: mul w2, w2, w1 0.00 : ffff8000101619b4: add x4, x4, #0x3 0.00 : ffff8000101619b8: add x2, x4, w2, sxtw : __write_once_size(): 0.00 : ffff8000101619bc: str x2, [x3, #9088] : print_cpu_stall(): : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff8000101619c0: mov x1, x0 0.00 : ffff8000101619c4: add x0, x20, #0xc80 0.00 : ffff8000101619c8: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : panic_on_rcu_stall(): : if (sysctl_panic_on_rcu_stall) 0.00 : ffff8000101619cc: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101619d0: ldr w0, [x0, #684] 0.00 : ffff8000101619d4: cbnz w0, ffff800010161d3c : get_current(): 0.00 : ffff8000101619d8: mrs x0, sp_el0 : arch_static_branch_jump(): 0.00 : ffff8000101619dc: b ffff800010161d34 0.00 : ffff8000101619e0: b ffff800010161d34 : __lse_atomic64_or(): : ATOMIC64_OP(or, stset) 0.00 : ffff8000101619e4: mov x1, #0x2 // #2 0.00 : ffff8000101619e8: stset x1, [x0] : get_current(): 0.00 : ffff8000101619ec: mrs x0, sp_el0 : set_preempt_need_resched(): 0.00 : ffff8000101619f0: str wzr, [x0, #20] 0.00 : ffff8000101619f4: b ffff80001016177c : rcu_print_task_stall(): : pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):", 0.00 : ffff8000101619f8: ldp w2, w3, [x4, #104] 0.00 : ffff8000101619fc: str x4, [x29, #120] 0.00 : ffff800010161a00: ldrb w1, [x4, #113] 0.00 : ffff800010161a04: ldr x0, [x29, #136] 0.00 : ffff800010161a08: bl ffff80001014a85c : t = list_entry(rnp->gp_tasks->prev, 0.00 : ffff800010161a0c: ldr x4, [x29, #120] 0.00 : ffff800010161a10: add x3, x4, #0x80 0.00 : ffff800010161a14: ldr x0, [x4, #144] : list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { 0.00 : ffff800010161a18: ldr x0, [x0, #8] 0.00 : ffff800010161a1c: ldr x2, [x0] 0.00 : ffff800010161a20: sub x2, x2, #0x310 0.00 : ffff800010161a24: add x0, x2, #0x310 0.00 : ffff800010161a28: cmp x0, x3 0.00 : ffff800010161a2c: b.ne ffff800010161ab0 // b.any 0.00 : ffff800010161a30: str x4, [x29, #120] : pr_cont("\n"); 0.00 : ffff800010161a34: adrp x0, ffff8000111b8000 0.00 : ffff800010161a38: add x0, x0, #0xb50 0.00 : ffff800010161a3c: bl ffff80001014a85c 0.00 : ffff800010161a40: ldr x4, [x29, #120] 0.00 : ffff800010161a44: b ffff80001016190c : print_other_cpu_stall(): : for_each_possible_cpu(cpu) 0.00 : ffff800010161a48: ldr x3, [x29, #152] 0.00 : ffff800010161a4c: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : rcu_get_n_cbs_cpu(): : struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); 0.00 : ffff800010161a50: adrp x2, ffff800011909000 0.00 : ffff800010161a54: add x5, x26, #0xac0 : print_other_cpu_stall(): 0.00 : ffff800010161a58: add x6, x3, #0x1a0 : rcu_get_n_cbs_cpu(): 0.00 : ffff800010161a5c: add x26, x2, #0x928 : print_other_cpu_stall(): 0.00 : ffff800010161a60: add x3, x1, #0x2f0 : long totqlen = 0; 0.00 : ffff800010161a64: mov x4, #0x0 // #0 : for_each_possible_cpu(cpu) 0.00 : ffff800010161a68: mov w0, #0xffffffff // #-1 0.00 : ffff800010161a6c: mov x1, x6 0.00 : ffff800010161a70: stp x5, x3, [x29, #136] 0.00 : ffff800010161a74: stp x4, x6, [x29, #152] 0.00 : ffff800010161a78: bl ffff800010cb92d8 0.00 : ffff800010161a7c: ldp x5, x3, [x29, #136] 0.00 : ffff800010161a80: ldp x4, x6, [x29, #152] 0.00 : ffff800010161a84: ldr w1, [x3] 0.00 : ffff800010161a88: cmp w0, w1 0.00 : ffff800010161a8c: b.cs ffff800010161aec // b.hs, b.nlast : rcu_get_n_cbs_cpu(): 0.00 : ffff800010161a90: ldr x2, [x26, w0, sxtw #3] 0.00 : ffff800010161a94: mov x1, x5 0.00 : ffff800010161a98: add x1, x1, x2 : if (rcu_segcblist_is_enabled(&rdp->cblist)) 0.00 : ffff800010161a9c: ldrb w2, [x1, #160] 0.00 : ffff800010161aa0: cbz w2, ffff800010161ae4 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010161aa4: ldr x1, [x1, #152] : print_other_cpu_stall(): : totqlen += rcu_get_n_cbs_cpu(cpu); 0.00 : ffff800010161aa8: add x4, x4, x1 0.00 : ffff800010161aac: b ffff800010161a6c : rcu_print_task_stall(): : ndetected++; 0.00 : ffff800010161ab0: ldr w0, [x29, #160] : pr_cont(" P%d", t->pid); 0.00 : ffff800010161ab4: ldr w1, [x2, #1128] : ndetected++; 0.00 : ffff800010161ab8: add w0, w0, #0x1 0.00 : ffff800010161abc: str w0, [x29, #160] : pr_cont(" P%d", t->pid); 0.00 : ffff800010161ac0: ldr x0, [x29, #128] 0.00 : ffff800010161ac4: stp x3, x4, [x29, #104] 0.00 : ffff800010161ac8: str x2, [x29, #120] 0.00 : ffff800010161acc: bl ffff80001014a85c : list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { 0.00 : ffff800010161ad0: ldr x2, [x29, #120] 0.00 : ffff800010161ad4: ldp x3, x4, [x29, #104] 0.00 : ffff800010161ad8: ldr x2, [x2, #784] 0.00 : ffff800010161adc: sub x2, x2, #0x310 0.00 : ffff800010161ae0: b ffff800010161a24 : rcu_get_n_cbs_cpu(): : return 0; 0.00 : ffff800010161ae4: mov x1, #0x0 // #0 0.00 : ffff800010161ae8: b ffff800010161aa8 : print_other_cpu_stall(): : pr_cont("\t(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n", 0.00 : ffff800010161aec: adrp x1, ffff80001151d000 0.00 : ffff800010161af0: add x1, x1, #0x18 : __read_once_size(): 0.00 : ffff800010161af4: add x26, x20, #0xc80 0.00 : ffff800010161af8: str x4, [x29, #160] : print_other_cpu_stall(): 0.00 : ffff800010161afc: bl ffff80001015c060 <__my_cpu_offset> 0.00 : ffff800010161b00: ldr x5, [x24, #2432] : __read_once_size(): 0.00 : ffff800010161b04: ldr x3, [x26, #8776] : print_other_cpu_stall(): 0.00 : ffff800010161b08: ldr w1, [x1, x0] 0.00 : ffff800010161b0c: adrp x0, ffff8000111c7000 0.00 : ffff800010161b10: ldr x4, [x29, #160] 0.00 : ffff800010161b14: add x0, x0, #0xb90 0.00 : ffff800010161b18: ldr x2, [x26, #9056] 0.00 : ffff800010161b1c: sub x2, x5, x2 0.00 : ffff800010161b20: bl ffff80001014a85c : if (ndetected) { 0.00 : ffff800010161b24: ldr w0, [x29, #168] 0.00 : ffff800010161b28: cbnz w0, ffff800010161be4 : __read_once_size(): 0.00 : ffff800010161b2c: ldr x0, [x26, #8776] : print_other_cpu_stall(): : if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) { 0.00 : ffff800010161b30: cmp x25, x0 0.00 : ffff800010161b34: b.eq ffff800010161ba8 // b.none : pr_err("INFO: Stall ended before state dump start\n"); 0.00 : ffff800010161b38: adrp x0, ffff8000111c7000 0.00 : ffff800010161b3c: add x0, x0, #0xbc8 0.00 : ffff800010161b40: bl ffff80001014a85c 0.00 : ffff800010161b44: adrp x12, ffff80001190a000 <__per_cpu_offset+0x6d8> : __read_once_size(): 0.00 : ffff800010161b48: add x1, x20, #0xc80 : print_other_cpu_stall(): : if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall))) 0.00 : ffff800010161b4c: ldr x0, [x24, #2432] : __read_once_size(): 0.00 : ffff800010161b50: ldr x1, [x1, #9088] : print_other_cpu_stall(): 0.00 : ffff800010161b54: sub x0, x0, x1 0.00 : ffff800010161b58: tbnz x0, #63, ffff800010161b8c : __read_once_size(): 0.00 : ffff800010161b5c: ldr w0, [x12, #660] : rcu_jiffies_till_stall_check(): : if (till_stall_check < 3) { 0.00 : ffff800010161b60: cmp w0, #0x2 0.00 : ffff800010161b64: b.gt ffff800010161c6c : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010161b68: mov w0, #0x3 // #3 0.00 : ffff800010161b6c: str w0, [x12, #660] : rcu_jiffies_till_stall_check(): : return till_stall_check * HZ + RCU_STALL_DELAY_DELTA; 0.00 : ffff800010161b70: mov w3, #0x2ee // #750 : print_other_cpu_stall(): : WRITE_ONCE(rcu_state.jiffies_stall, 0.00 : ffff800010161b74: ldr x2, [x24, #2432] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010161b78: add x1, x20, #0xc80 : print_other_cpu_stall(): 0.00 : ffff800010161b7c: mul w0, w0, w3 0.00 : ffff800010161b80: add x2, x2, #0x3 0.00 : ffff800010161b84: add x0, x2, w0, sxtw : __write_once_size(): 0.00 : ffff800010161b88: str x0, [x1, #9088] : print_other_cpu_stall(): : rcu_check_gp_kthread_starvation(); 0.00 : ffff800010161b8c: bl ffff8000101626fc : panic_on_rcu_stall(): : if (sysctl_panic_on_rcu_stall) 0.00 : ffff800010161b90: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010161b94: add x0, x0, #0x2a0 0.00 : ffff800010161b98: ldr w0, [x0, #12] 0.00 : ffff800010161b9c: cbnz w0, ffff800010161d3c : print_other_cpu_stall(): : rcu_force_quiescent_state(); /* Kick them all. */ 0.00 : ffff800010161ba0: bl ffff80001015e1e0 0.00 : ffff800010161ba4: b ffff80001016150c : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010161ba8: adrp x0, ffff800011922000 0.00 : ffff800010161bac: add x0, x0, #0xb00 : print_other_cpu_stall(): : j = jiffies; 0.00 : ffff800010161bb0: ldr x3, [x24, #2432] : __read_once_size(): 0.00 : ffff800010161bb4: ldr x4, [x26, #9072] : print_other_cpu_stall(): : pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n", 0.00 : ffff800010161bb8: ldr x6, [x26, #32] : __read_once_size(): 0.00 : ffff800010161bbc: ldr x5, [x0, #32] : print_other_cpu_stall(): 0.00 : ffff800010161bc0: sub x2, x3, x4 0.00 : ffff800010161bc4: ldr x1, [x26, #9120] 0.00 : ffff800010161bc8: adrp x0, ffff8000111c7000 0.00 : ffff800010161bcc: add x0, x0, #0xc00 0.00 : ffff800010161bd0: bl ffff80001014a85c : get_current(): 0.00 : ffff800010161bd4: mrs x0, sp_el0 : print_other_cpu_stall(): : sched_show_task(current); 0.00 : ffff800010161bd8: bl ffff800010113ef8 0.00 : ffff800010161bdc: adrp x12, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010161be0: b ffff800010161b48 : rcu_dump_cpu_stacks(); 0.00 : ffff800010161be4: bl ffff8000101627cc : rcu_for_each_leaf_node(rnp) 0.00 : ffff800010161be8: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010161bec: add x0, x0, #0x2a0 0.00 : ffff800010161bf0: stp x26, x0, [x29, #160] 0.00 : ffff800010161bf4: adrp x12, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010161bf8: ldr w0, [x0, #4] 0.00 : ffff800010161bfc: sub w0, w0, #0x1 0.00 : ffff800010161c00: add x26, x26, w0, sxtw #3 0.00 : ffff800010161c04: ldr x25, [x26, #8704] 0.00 : ffff800010161c08: ldp x1, x0, [x29, #160] 0.00 : ffff800010161c0c: ldrsw x0, [x0] 0.00 : ffff800010161c10: add x0, x1, x0, lsl #9 0.00 : ffff800010161c14: cmp x25, x0 0.00 : ffff800010161c18: b.cs ffff800010161b48 // b.hs, b.nlast : rcu_print_detail_task_stall_rnp(): : raw_spin_lock_irqsave_rcu_node(rnp, flags); 0.00 : ffff800010161c1c: mov x0, x25 0.00 : ffff800010161c20: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff800010161c24: mov x1, x0 : __read_once_size(): 0.00 : ffff800010161c28: ldr x0, [x25, #144] : rcu_print_detail_task_stall_rnp(): : if (!rcu_preempt_blocked_readers_cgp(rnp)) { 0.00 : ffff800010161c2c: cbz x0, ffff800010161c84 : t = list_entry(rnp->gp_tasks->prev, 0.00 : ffff800010161c30: ldr x0, [x25, #144] 0.00 : ffff800010161c34: add x2, x25, #0x80 : list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { 0.00 : ffff800010161c38: ldr x0, [x0, #8] 0.00 : ffff800010161c3c: ldr x26, [x0] 0.00 : ffff800010161c40: sub x26, x26, #0x310 0.00 : ffff800010161c44: add x0, x26, #0x310 0.00 : ffff800010161c48: cmp x2, x0 0.00 : ffff800010161c4c: b.eq ffff800010161c84 // b.none : sched_show_task(t); 0.00 : ffff800010161c50: mov x0, x26 0.00 : ffff800010161c54: stp x2, x1, [x29, #144] 0.00 : ffff800010161c58: bl ffff800010113ef8 : list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { 0.00 : ffff800010161c5c: ldr x26, [x26, #784] 0.00 : ffff800010161c60: ldp x2, x1, [x29, #144] 0.00 : ffff800010161c64: sub x26, x26, #0x310 0.00 : ffff800010161c68: b ffff800010161c44 : rcu_jiffies_till_stall_check(): : } else if (till_stall_check > 300) { 0.00 : ffff800010161c6c: cmp w0, #0x12c 0.00 : ffff800010161c70: b.le ffff800010161b70 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010161c74: mov w1, #0x12c // #300 : rcu_jiffies_till_stall_check(): : till_stall_check = 300; 0.00 : ffff800010161c78: mov w0, w1 : __write_once_size(): 0.00 : ffff800010161c7c: str w1, [x12, #660] 0.00 : ffff800010161c80: b ffff800010161b70 : rcu_print_detail_task_stall_rnp(): : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff800010161c84: mov x0, x25 0.00 : ffff800010161c88: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010161c8c: adrp x12, ffff80001190a000 <__per_cpu_offset+0x6d8> : print_other_cpu_stall(): : rcu_for_each_leaf_node(rnp) 0.00 : ffff800010161c90: add x25, x25, #0x200 0.00 : ffff800010161c94: b ffff800010161c08 : for_each_leaf_node_possible_cpu(rnp, cpu) 0.00 : ffff800010161c98: ldr w0, [x4, #104] 0.00 : ffff800010161c9c: ldr x1, [x29, #152] 0.00 : ffff800010161ca0: str x4, [x29, #120] 0.00 : ffff800010161ca4: sub w0, w0, #0x1 0.00 : ffff800010161ca8: add x1, x1, #0x1a0 0.00 : ffff800010161cac: str x1, [x29, #160] 0.00 : ffff800010161cb0: bl ffff800010cb92d8 0.00 : ffff800010161cb4: mov w1, w0 0.00 : ffff800010161cb8: ldr x4, [x29, #120] 0.00 : ffff800010161cbc: ldr w0, [x4, #108] 0.00 : ffff800010161cc0: cmp w1, w0 0.00 : ffff800010161cc4: b.gt ffff800010161924 : if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { 0.00 : ffff800010161cc8: ldr w0, [x4, #104] 0.00 : ffff800010161ccc: sub w2, w1, w0 0.00 : ffff800010161cd0: ldr x0, [x4, #32] 0.00 : ffff800010161cd4: lsr x0, x0, x2 0.00 : ffff800010161cd8: tbz w0, #0, ffff800010161d00 : ndetected++; 0.00 : ffff800010161cdc: ldr w0, [x29, #168] 0.00 : ffff800010161ce0: str x4, [x29, #112] 0.00 : ffff800010161ce4: add w0, w0, #0x1 : print_cpu_stall_info(cpu); 0.00 : ffff800010161ce8: str w1, [x29, #120] : ndetected++; 0.00 : ffff800010161cec: str w0, [x29, #168] : print_cpu_stall_info(cpu); 0.00 : ffff800010161cf0: mov w0, w1 0.00 : ffff800010161cf4: bl ffff8000101628ac 0.00 : ffff800010161cf8: ldr w1, [x29, #120] 0.00 : ffff800010161cfc: ldr x4, [x29, #112] : for_each_leaf_node_possible_cpu(rnp, cpu) 0.00 : ffff800010161d00: mov w0, w1 0.00 : ffff800010161d04: ldr x1, [x29, #160] 0.00 : ffff800010161d08: str x4, [x29, #120] 0.00 : ffff800010161d0c: bl ffff800010cb92d8 0.00 : ffff800010161d10: mov w1, w0 0.00 : ffff800010161d14: ldr x4, [x29, #120] 0.00 : ffff800010161d18: b ffff800010161cbc : rcu_jiffies_till_stall_check(): : } else if (till_stall_check > 300) { 0.00 : ffff800010161d1c: cmp w2, #0x12c 0.00 : ffff800010161d20: b.le ffff8000101619a4 : __write_once_size(): 0.00 : ffff800010161d24: mov w3, #0x12c // #300 : rcu_jiffies_till_stall_check(): : till_stall_check = 300; 0.00 : ffff800010161d28: mov w2, w3 : __write_once_size(): 0.00 : ffff800010161d2c: str w3, [x12, #660] 0.00 : ffff800010161d30: b ffff8000101619a4 : __ll_sc_atomic64_or(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010161d34: b ffff8000101626cc 0.00 : ffff800010161d38: b ffff8000101619ec : panic_on_rcu_stall(): : panic("RCU Stall\n"); 0.00 : ffff800010161d3c: adrp x0, ffff8000111c7000 0.00 : ffff800010161d40: add x0, x0, #0x8a8 0.00 : ffff800010161d44: bl ffff8000100e67ac Percent | Source code & Disassembly of vmlinux for cycles (203 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc8aa0 : : timerqueue_add(): : * Adds the timer node to the timerqueue, sorted by the node's expires : * value. Returns true if the newly added timer is the first expiring timer in : * the queue. : */ : bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node) : { 2.98 : ffff800010cc8aa0: stp x29, x30, [sp, #-32]! 0.00 : ffff800010cc8aa4: mov x4, x1 0.00 : ffff800010cc8aa8: mov x29, sp 9.20 : ffff800010cc8aac: str x19, [sp, #16] : struct rb_node *parent = NULL; : struct timerqueue_node *ptr; : bool leftmost = true; : : /* Make sure we don't add nodes that are already added */ : WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node)); 0.94 : ffff800010cc8ab0: ldr x1, [x1] 0.00 : ffff800010cc8ab4: cmp x4, x1 0.00 : ffff800010cc8ab8: b.ne ffff800010cc8b30 // b.any : struct rb_node **p = &head->rb_root.rb_root.rb_node; 0.00 : ffff800010cc8abc: mov x3, x0 : bool leftmost = true; 0.50 : ffff800010cc8ac0: mov w19, #0x1 // #1 : struct rb_node *parent = NULL; 0.00 : ffff800010cc8ac4: mov x5, #0x0 // #0 : : while (*p) { 0.52 : ffff800010cc8ac8: ldr x2, [x3] 0.00 : ffff800010cc8acc: cbz x2, ffff800010cc8af4 : parent = *p; : ptr = rb_entry(parent, struct timerqueue_node, node); : if (node->expires < ptr->expires) { 2.97 : ffff800010cc8ad0: ldr x5, [x2, #24] : p = &(*p)->rb_left; : } else { : p = &(*p)->rb_right; 0.00 : ffff800010cc8ad4: add x3, x2, #0x8 : if (node->expires < ptr->expires) { 8.97 : ffff800010cc8ad8: ldr x6, [x4, #24] 0.00 : ffff800010cc8adc: cmp x6, x5 0.00 : ffff800010cc8ae0: b.ge ffff800010cc8b24 // b.tcont : p = &(*p)->rb_left; 0.00 : ffff800010cc8ae4: add x3, x2, #0x10 : leftmost = false; 0.00 : ffff800010cc8ae8: mov x5, x2 : while (*p) { 2.00 : ffff800010cc8aec: ldr x2, [x3] 0.00 : ffff800010cc8af0: cbnz x2, ffff800010cc8ad0 : rb_link_node(): : : static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, : struct rb_node **rb_link) : { : node->__rb_parent_color = (unsigned long)parent; : node->rb_left = node->rb_right = NULL; 6.41 : ffff800010cc8af4: stp x5, xzr, [x4] 3.95 : ffff800010cc8af8: str xzr, [x4, #16] : : *rb_link = node; 0.00 : ffff800010cc8afc: str x4, [x3] : rb_insert_color_cached(): : : static inline void rb_insert_color_cached(struct rb_node *node, : struct rb_root_cached *root, : bool leftmost) : { : if (leftmost) 0.00 : ffff800010cc8b00: cbz w19, ffff800010cc8b08 : root->rb_leftmost = node; 0.00 : ffff800010cc8b04: str x4, [x0, #8] : rb_insert_color(node, &root->rb_root); 0.00 : ffff800010cc8b08: mov x1, x0 0.00 : ffff800010cc8b0c: mov x0, x4 0.00 : ffff800010cc8b10: bl ffff800010cc4ca0 : timerqueue_add(): : } : rb_link_node(&node->node, parent, p); : rb_insert_color_cached(&node->node, &head->rb_root, leftmost); : : return leftmost; : } 6.27 : ffff800010cc8b14: mov w0, w19 24.47 : ffff800010cc8b18: ldr x19, [sp, #16] 30.83 : ffff800010cc8b1c: ldp x29, x30, [sp], #32 0.00 : ffff800010cc8b20: ret : leftmost = false; 0.00 : ffff800010cc8b24: mov w19, #0x0 // #0 0.00 : ffff800010cc8b28: mov x5, x2 0.00 : ffff800010cc8b2c: b ffff800010cc8aec : WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node)); 0.00 : ffff800010cc8b30: brk #0x800 0.00 : ffff800010cc8b34: b ffff800010cc8abc Percent | Source code & Disassembly of vmlinux for cycles (198 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016b5b8 : : hrtimer_reprogram(): : * which the clock event device was armed. : * : * Called with interrupts disabled and base->cpu_base.lock held : */ : static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram) : { 0.00 : ffff80001016b5b8: stp x29, x30, [sp, #-16]! : struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016b5bc: adrp x2, ffff800011525000 : { 3.05 : ffff80001016b5c0: and w1, w1, #0xff : struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016b5c4: add x2, x2, #0x980 : { 8.61 : ffff80001016b5c8: mov x29, sp : struct hrtimer_clock_base *base = timer->base; 0.50 : ffff80001016b5cc: ldr x5, [x0, #48] : ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset); 12.86 : ffff80001016b5d0: ldr x4, [x0, #24] 5.19 : ffff80001016b5d4: ldr x3, [x5, #56] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 16.08 : ffff80001016b5d8: mrs x7, tpidr_el1 : hrtimer_reprogram(): 0.00 : ffff80001016b5dc: sub x3, x4, x3 : : WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); 0.00 : ffff80001016b5e0: tbnz x4, #63, ffff80001016b66c : * expiry time which is less than base->offset. Set it to 0. : */ : if (expires < 0) : expires = 0; : : if (timer->is_soft) { 1.02 : ffff80001016b5e4: ldrb w6, [x0, #58] 0.00 : ffff80001016b5e8: cmp x3, #0x0 0.00 : ffff80001016b5ec: csel x3, x3, xzr, ge // ge = tcont 0.00 : ffff80001016b5f0: ldr x4, [x5] 0.00 : ffff80001016b5f4: cbz w6, ffff80001016b630 : * first hard hrtimer on the remote CPU - : * hrtimer_check_target() prevents this case. : */ : struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base; : : if (timer_cpu_base->softirq_activated) 0.00 : ffff80001016b5f8: ldrb w6, [x4, #16] 0.00 : ffff80001016b5fc: tbnz w6, #3, ffff80001016b620 : ktime_compare(): : * cmp1 == cmp2: return 0 : * cmp1 > cmp2: return >0 : */ : static inline int ktime_compare(const ktime_t cmp1, const ktime_t cmp2) : { : if (cmp1 < cmp2) 0.00 : ffff80001016b600: ldr x6, [x4, #48] 0.00 : ffff80001016b604: cmp x6, x3 0.00 : ffff80001016b608: b.le ffff80001016b620 : hrtimer_reprogram(): : return; : : timer_cpu_base->softirq_next_timer = timer; : timer_cpu_base->softirq_expires_next = expires; : : if (!ktime_before(expires, timer_cpu_base->expires_next) || 0.00 : ffff80001016b60c: ldr x6, [x4, #32] 0.00 : ffff80001016b610: cmp w1, #0x0 : timer_cpu_base->softirq_next_timer = timer; 0.00 : ffff80001016b614: stp x3, x0, [x4, #48] : if (!ktime_before(expires, timer_cpu_base->expires_next) || 0.00 : ffff80001016b618: ccmp x6, x3, #0x4, ne // ne = any 0.00 : ffff80001016b61c: b.gt ffff80001016b628 : /* : * Program the timer hardware. We enforce the expiry for : * events which are already in the past. : */ : tick_program_event(expires, 1); : } 5.49 : ffff80001016b620: ldp x29, x30, [sp], #16 0.00 : ffff80001016b624: ret 0.00 : ffff80001016b628: ldr x4, [x5] 0.00 : ffff80001016b62c: nop : struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 5.96 : ffff80001016b630: add x2, x2, x7 : if (base->cpu_base != cpu_base) 0.00 : ffff80001016b634: cmp x2, x4 0.00 : ffff80001016b638: b.ne ffff80001016b620 // b.any : if (cpu_base->in_hrtirq) 0.00 : ffff80001016b63c: ldrb w1, [x4, #16] 0.00 : ffff80001016b640: tbnz w1, #1, ffff80001016b620 : if (expires >= cpu_base->expires_next) 26.10 : ffff80001016b644: ldr x2, [x4, #32] 0.00 : ffff80001016b648: cmp x2, x3 0.00 : ffff80001016b64c: b.le ffff80001016b620 : cpu_base->next_timer = timer; 6.95 : ffff80001016b650: stp x3, x0, [x4, #32] : if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected) 0.00 : ffff80001016b654: tbz w1, #0, ffff80001016b620 0.52 : ffff80001016b658: tbnz w1, #2, ffff80001016b620 : tick_program_event(expires, 1); 7.68 : ffff80001016b65c: mov w1, #0x1 // #1 0.00 : ffff80001016b660: mov x0, x3 0.00 : ffff80001016b664: bl ffff80001017afa8 0.00 : ffff80001016b668: b ffff80001016b620 : WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); 0.00 : ffff80001016b66c: brk #0x800 0.00 : ffff80001016b670: b ffff80001016b5e4 Percent | Source code & Disassembly of vmlinux for cycles (197 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016e578 : : timekeeping_max_deferment(): : : /** : * timekeeping_max_deferment - Returns max time the clocksource can be deferred : */ : u64 timekeeping_max_deferment(void) : { 1.04 : ffff80001016e578: adrp x1, ffff800011b20000 <__log_buf+0x1fc10> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001016e57c: add x1, x1, #0xec0 0.00 : ffff80001016e580: ldr w2, [x1, #64] : __read_seqcount_begin(): : { : unsigned ret; : : repeat: : ret = READ_ONCE(s->sequence); : if (unlikely(ret & 1)) { 0.00 : ffff80001016e584: tbnz w2, #0, ffff80001016e5a8 : raw_read_seqcount_begin(): : * section is tested by checking read_seqcount_retry function. : */ : static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) : { : unsigned ret = __read_seqcount_begin(s); : smp_rmb(); 18.91 : ffff80001016e588: dmb ishld : timekeeping_max_deferment(): : u64 ret; : : do { : seq = read_seqcount_begin(&tk_core.seq); : : ret = tk->tkr_mono.clock->max_idle_ns; 43.91 : ffff80001016e58c: ldr x0, [x1, #72] 0.00 : ffff80001016e590: ldr x0, [x0, #24] : read_seqcount_retry(): : * If the critical section was invalid, it must be ignored (and typically : * retried). : */ : static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) : { : smp_rmb(); 0.99 : ffff80001016e594: dmb ishld : timekeeping_max_deferment(): : : } while (read_seqcount_retry(&tk_core.seq, seq)); 35.15 : ffff80001016e598: ldr w3, [x1, #64] 0.00 : ffff80001016e59c: cmp w3, w2 0.00 : ffff80001016e5a0: b.ne ffff80001016e580 // b.any : : return ret; : } 0.00 : ffff80001016e5a4: ret : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff80001016e5a8: yield 0.00 : ffff80001016e5ac: b ffff80001016e580 Percent | Source code & Disassembly of vmlinux for cycles (187 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010127d18 : : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010127d18: ldr x1, [x0, #2472] : trigger_load_balance(): : * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. : */ : void trigger_load_balance(struct rq *rq) : { : /* Don't need to rebalance while attached to NULL domain */ : if (unlikely(on_null_domain(rq))) 0.00 : ffff800010127d1c: cbz x1, ffff800010127d8c : { 0.00 : ffff800010127d20: stp x29, x30, [sp, #-96]! 0.00 : ffff800010127d24: mov x29, sp 0.00 : ffff800010127d28: stp x19, x20, [sp, #16] : return; : : if (time_after_eq(jiffies, rq->next_balance)) 0.00 : ffff800010127d2c: adrp x20, ffff800011907000 0.00 : ffff800010127d30: mov x19, x0 : { 0.00 : ffff800010127d34: stp x21, x22, [sp, #32] : if (time_after_eq(jiffies, rq->next_balance)) 0.00 : ffff800010127d38: ldr x0, [x20, #2432] 22.12 : ffff800010127d3c: ldr x1, [x19, #2376] 0.00 : ffff800010127d40: sub x0, x0, x1 0.00 : ffff800010127d44: tbz x0, #63, ffff800010127d80 : nohz_balancer_kick(): : unsigned long now = jiffies; 0.00 : ffff800010127d48: ldr x21, [x20, #2432] : if (unlikely(rq->idle_balance)) 0.00 : ffff800010127d4c: ldrb w0, [x19, #2504] : int nr_busy, i, cpu = rq->cpu; 0.00 : ffff800010127d50: ldr w22, [x19, #2568] : if (unlikely(rq->idle_balance)) 0.00 : ffff800010127d54: cbnz w0, ffff800010127d70 : nohz_balance_exit_idle(): : if (likely(!rq->nohz_tick_stopped)) 0.00 : ffff800010127d58: ldr w0, [x19, #44] 0.00 : ffff800010127d5c: cbnz w0, ffff800010127d90 : __read_once_size(): 0.00 : ffff800010127d60: adrp x20, ffff800011afb000 0.00 : ffff800010127d64: add x0, x20, #0xfc0 0.00 : ffff800010127d68: ldr w1, [x0, #32] : nohz_balancer_kick(): : if (likely(!atomic_read(&nohz.nr_cpus))) 0.00 : ffff800010127d6c: cbnz w1, ffff800010127d9c : trigger_load_balance(): : raise_softirq(SCHED_SOFTIRQ); : : nohz_balancer_kick(rq); : } 0.54 : ffff800010127d70: ldp x19, x20, [sp, #16] 0.00 : ffff800010127d74: ldp x21, x22, [sp, #32] 0.00 : ffff800010127d78: ldp x29, x30, [sp], #96 0.00 : ffff800010127d7c: ret : raise_softirq(SCHED_SOFTIRQ); 0.00 : ffff800010127d80: mov w0, #0x7 // #7 0.00 : ffff800010127d84: bl ffff8000100ecde0 0.00 : ffff800010127d88: b ffff800010127d48 0.00 : ffff800010127d8c: ret : nohz_balance_exit_idle(): 0.49 : ffff800010127d90: mov x0, x19 0.00 : ffff800010127d94: bl ffff80001011e188 0.00 : ffff800010127d98: b ffff800010127d60 65.52 : ffff800010127d9c: str x23, [x29, #48] : __read_once_size(): 1.61 : ffff800010127da0: ldr w23, [x0, #36] 2.19 : ffff800010127da4: ldr x1, [x0, #40] 0.00 : ffff800010127da8: sub x1, x21, x1 : nohz_balancer_kick(): : if (READ_ONCE(nohz.has_blocked) && 0.00 : ffff800010127dac: cbnz w23, ffff800010127dd4 : if (time_before(now, nohz.next_balance)) 0.00 : ffff800010127db0: tbnz x1, #63, ffff800010127dec : if (rq->nr_running >= 2) { 0.00 : ffff800010127db4: ldr w0, [x19, #4] 0.00 : ffff800010127db8: cmp w0, #0x1 0.00 : ffff800010127dbc: b.ls ffff800010127e00 // b.plast : flags = NOHZ_KICK_MASK; 0.00 : ffff800010127dc0: mov w23, #0x3 // #3 : kick_ilb(flags); 0.00 : ffff800010127dc4: mov w0, w23 0.00 : ffff800010127dc8: bl ffff80001011e090 0.00 : ffff800010127dcc: ldr x23, [x29, #48] 0.00 : ffff800010127dd0: b ffff800010127d70 : __read_once_size(): 0.55 : ffff800010127dd4: ldr x0, [x0, #48] : nohz_balancer_kick(): : time_after(now, READ_ONCE(nohz.next_blocked))) 0.00 : ffff800010127dd8: sub x21, x0, x21 0.00 : ffff800010127ddc: tbnz x21, #63, ffff800010127df4 : if (time_before(now, nohz.next_balance)) 2.69 : ffff800010127de0: tbnz x1, #63, ffff800010127dec : unsigned int flags = 0; 0.00 : ffff800010127de4: mov w23, #0x0 // #0 0.00 : ffff800010127de8: b ffff800010127db4 1.64 : ffff800010127dec: ldr x23, [x29, #48] 0.00 : ffff800010127df0: b ffff800010127d70 : flags = NOHZ_STATS_KICK; 0.00 : ffff800010127df4: mov w23, #0x2 // #2 : if (time_before(now, nohz.next_balance)) 0.00 : ffff800010127df8: tbz x1, #63, ffff800010127db4 0.00 : ffff800010127dfc: b ffff800010127dc4 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff800010127e00: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.00 : ffff800010127e04: ldr x0, [x19, #2472] : nohz_balancer_kick(): : if (sd) { 0.00 : ffff800010127e08: cbz x0, ffff800010127e34 : if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) { 0.00 : ffff800010127e0c: ldr w1, [x19, #156] 0.00 : ffff800010127e10: cbz w1, ffff800010127e34 : check_cpu_capacity(): : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010127e14: ldr w0, [x0, #44] 0.00 : ffff800010127e18: ldr x3, [x19, #2480] : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010127e1c: ldr x2, [x19, #2488] 0.00 : ffff800010127e20: add x1, x2, x2, lsl #1 : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010127e24: mul x0, x0, x3 : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010127e28: add x1, x2, x1, lsl #3 : nohz_balancer_kick(): : if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) { 0.00 : ffff800010127e2c: cmp x0, x1, lsl #2 0.00 : ffff800010127e30: b.cc ffff800010127f0c // b.lo, b.ul, b.last 0.00 : ffff800010127e34: stp x24, x25, [x29, #56] : sd = rcu_dereference(per_cpu(sd_asym_packing, cpu)); 0.00 : ffff800010127e38: adrp x25, ffff800011909000 0.00 : ffff800010127e3c: str x26, [x29, #72] 0.00 : ffff800010127e40: add x1, x25, #0x928 0.00 : ffff800010127e44: sxtw x26, w22 0.00 : ffff800010127e48: adrp x0, ffff80001151f000 0.00 : ffff800010127e4c: add x0, x0, #0x138 0.00 : ffff800010127e50: ldr x1, [x1, x26, lsl #3] : __read_once_size(): 0.00 : ffff800010127e54: ldr x24, [x0, x1] : nohz_balancer_kick(): : if (sd) { 0.00 : ffff800010127e58: cbz x24, ffff800010127ebc 0.00 : ffff800010127e5c: stp x27, x28, [x29, #80] 0.00 : ffff800010127e60: add x24, x24, #0x88 0.00 : ffff800010127e64: adrp x27, ffff80001190a000 <__per_cpu_offset+0x6d8> : for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 0.00 : ffff800010127e68: mov w21, #0xffffffff // #-1 0.00 : ffff800010127e6c: add x20, x20, #0xfc0 0.00 : ffff800010127e70: add x27, x27, #0x2f0 0.00 : ffff800010127e74: b ffff800010127e90 : sched_asym_prefer(): : return scale_load_down(se->runnable_weight); : } : : static inline bool sched_asym_prefer(int a, int b) : { : return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b); 0.00 : ffff800010127e78: bl ffff800010123c30 0.00 : ffff800010127e7c: mov w28, w0 0.00 : ffff800010127e80: mov w0, w22 0.00 : ffff800010127e84: bl ffff800010123c30 : nohz_balancer_kick(): : if (sched_asym_prefer(i, cpu)) { 0.00 : ffff800010127e88: cmp w28, w0 0.00 : ffff800010127e8c: b.gt ffff800010127f48 : for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 0.00 : ffff800010127e90: mov x1, x24 0.00 : ffff800010127e94: mov w0, w21 0.00 : ffff800010127e98: mov x2, x20 0.00 : ffff800010127e9c: bl ffff800010cb93f0 0.00 : ffff800010127ea0: ldr w1, [x27] 0.00 : ffff800010127ea4: mov w21, w0 0.00 : ffff800010127ea8: cmp w0, w1 0.00 : ffff800010127eac: b.cc ffff800010127e78 // b.lo, b.ul, b.last 0.00 : ffff800010127eb0: add x25, x25, #0x928 0.00 : ffff800010127eb4: ldp x27, x28, [x29, #80] 0.00 : ffff800010127eb8: ldr x1, [x25, x26, lsl #3] : sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu)); 0.00 : ffff800010127ebc: adrp x0, ffff80001151f000 0.00 : ffff800010127ec0: add x0, x0, #0x140 : __read_once_size(): 0.00 : ffff800010127ec4: ldr x0, [x1, x0] : nohz_balancer_kick(): : if (sd) { 0.00 : ffff800010127ec8: cbz x0, ffff800010127f18 : check_misfit_status(): : return rq->misfit_task_load && 0.00 : ffff800010127ecc: ldr x1, [x19, #2512] 0.00 : ffff800010127ed0: cbz x1, ffff800010127f34 : (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity || 0.00 : ffff800010127ed4: ldr x2, [x19, #2464] 0.00 : ffff800010127ed8: ldr x1, [x19, #2488] : return rq->misfit_task_load && 0.00 : ffff800010127edc: ldr x2, [x2, #4376] 0.00 : ffff800010127ee0: cmp x1, x2 0.00 : ffff800010127ee4: b.cc ffff800010127f04 // b.lo, b.ul, b.last : check_cpu_capacity(): : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010127ee8: ldr w0, [x0, #44] : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010127eec: add x2, x1, x1, lsl #1 : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010127ef0: ldr x3, [x19, #2480] : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010127ef4: add x1, x1, x2, lsl #3 : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010127ef8: mul x0, x0, x3 : check_misfit_status(): : (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity || 0.00 : ffff800010127efc: cmp x0, x1, lsl #2 0.00 : ffff800010127f00: b.cs ffff800010127f34 // b.hs, b.nlast 2.64 : ffff800010127f04: ldp x24, x25, [x29, #56] 0.00 : ffff800010127f08: ldr x26, [x29, #72] : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff800010127f0c: bl ffff800010160d20 <__rcu_read_unlock> : nohz_balancer_kick(): : flags = NOHZ_KICK_MASK; 0.00 : ffff800010127f10: mov w23, #0x3 // #3 0.00 : ffff800010127f14: b ffff800010127dc4 : sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 0.00 : ffff800010127f18: adrp x0, ffff80001151f000 0.00 : ffff800010127f1c: add x0, x0, #0x128 : __read_once_size(): 0.00 : ffff800010127f20: ldr x0, [x1, x0] : nohz_balancer_kick(): : if (sds) { 0.00 : ffff800010127f24: cbz x0, ffff800010127f34 : __read_once_size(): 0.00 : ffff800010127f28: ldr w0, [x0, #4] : nohz_balancer_kick(): : if (nr_busy > 1) { 0.00 : ffff800010127f2c: cmp w0, #0x1 0.00 : ffff800010127f30: b.gt ffff800010127f04 : rcu_read_unlock(): 0.00 : ffff800010127f34: bl ffff800010160d20 <__rcu_read_unlock> : nohz_balancer_kick(): : if (flags) 0.00 : ffff800010127f38: cbz w23, ffff800010127f58 0.00 : ffff800010127f3c: ldp x24, x25, [x29, #56] 0.00 : ffff800010127f40: ldr x26, [x29, #72] 0.00 : ffff800010127f44: b ffff800010127dc4 0.00 : ffff800010127f48: ldp x24, x25, [x29, #56] 0.00 : ffff800010127f4c: ldp x26, x27, [x29, #72] 0.00 : ffff800010127f50: ldr x28, [x29, #88] 0.00 : ffff800010127f54: b ffff800010127f0c 0.00 : ffff800010127f58: ldp x23, x24, [x29, #48] 0.00 : ffff800010127f5c: ldp x25, x26, [x29, #64] 0.00 : ffff800010127f60: b ffff800010127d70 Percent | Source code & Disassembly of vmlinux for cycles (181 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010711948 : : iommu_unmap_fast(): : EXPORT_SYMBOL_GPL(iommu_unmap); : : size_t iommu_unmap_fast(struct iommu_domain *domain, : unsigned long iova, size_t size, : struct iommu_iotlb_gather *iotlb_gather) : { 16.69 : ffff800010711948: stp x29, x30, [sp, #-16]! 5.86 : ffff80001071194c: mov x29, sp : return __iommu_unmap(domain, iova, size, iotlb_gather); 11.70 : ffff800010711950: bl ffff8000107117a8 <__iommu_unmap> : } 65.75 : ffff800010711954: ldp x29, x30, [sp], #16 0.00 : ffff800010711958: ret Percent | Source code & Disassembly of vmlinux for cycles (387 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010160440 : : rcu_note_context_switch(): : * rnp->gp_tasks becomes NULL. : * : * Caller must disable interrupts. : */ : void rcu_note_context_switch(bool preempt) : { 4.61 : ffff800010160440: stp x29, x30, [sp, #-48]! : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); : struct rcu_node *rnp; : : trace_rcu_utilization(TPS("Start context switch")); : lockdep_assert_irqs_disabled(); : WARN_ON_ONCE(!preempt && rcu_preempt_depth() > 0); 0.00 : ffff800010160444: tst w0, #0xff : { 0.00 : ffff800010160448: mov x29, sp 50.85 : ffff80001016044c: str x19, [sp, #16] : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff800010160450: adrp x19, ffff800011529000 : { 2.62 : ffff800010160454: str x21, [sp, #32] : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff800010160458: add x19, x19, #0xac0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001016045c: mrs x0, tpidr_el1 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010160460: mrs x21, sp_el0 : rcu_note_context_switch(): 0.00 : ffff800010160464: add x19, x19, x0 : WARN_ON_ONCE(!preempt && rcu_preempt_depth() > 0); 0.00 : ffff800010160468: b.ne ffff80001016047c // b.any 3.08 : ffff80001016046c: ldr w0, [x21, #776] 0.00 : ffff800010160470: cmp w0, #0x0 0.00 : ffff800010160474: b.le ffff80001016047c 0.00 : ffff800010160478: brk #0x800 : get_current(): 18.69 : ffff80001016047c: mrs x0, sp_el0 : rcu_note_context_switch(): : if (rcu_preempt_depth() > 0 && 3.03 : ffff800010160480: ldr w0, [x0, #776] 0.00 : ffff800010160484: cmp w0, #0x0 0.00 : ffff800010160488: b.le ffff800010160494 0.00 : ffff80001016048c: ldrb w0, [x21, #780] 0.00 : ffff800010160490: cbz w0, ffff8000101604c8 : (rnp->qsmask & rdp->grpmask) : ? rnp->gp_seq : : rcu_seq_snap(&rnp->gp_seq)); : rcu_preempt_ctxt_queue(rnp, rdp); : } else { : rcu_preempt_deferred_qs(t); 5.43 : ffff800010160494: mov x0, x21 0.00 : ffff800010160498: bl ffff800010160210 : * for this CPU. Again, if we were in an RCU read-side critical : * section, and if that critical section was blocking the current : * grace period, then the fact that the task has been enqueued : * means that we continue to block the current grace period. : */ : rcu_qs(); 3.34 : ffff80001016049c: bl ffff80001015df08 : if (rdp->exp_deferred_qs) 3.12 : ffff8000101604a0: ldrb w0, [x19, #21] 0.00 : ffff8000101604a4: cbz w0, ffff8000101604b8 : rcu_report_exp_rdp(): : * Report expedited quiescent state for specified rcu_data (CPU). : */ : static void rcu_report_exp_rdp(struct rcu_data *rdp) : { : WRITE_ONCE(rdp->exp_deferred_qs, false); : rcu_report_exp_cpu_mult(rdp->mynode, rdp->grpmask, true); 0.00 : ffff8000101604a8: ldp x0, x1, [x19, #24] : __write_once_size(): : } : : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff8000101604ac: strb wzr, [x19, #21] : rcu_report_exp_rdp(): 0.00 : ffff8000101604b0: mov w2, #0x1 // #1 0.00 : ffff8000101604b4: bl ffff80001015c340 : rcu_note_context_switch(): : rcu_report_exp_rdp(rdp); : trace_rcu_utilization(TPS("End context switch")); : } 0.27 : ffff8000101604b8: ldr x19, [sp, #16] 0.27 : ffff8000101604bc: ldr x21, [sp, #32] 0.00 : ffff8000101604c0: ldp x29, x30, [sp], #48 0.00 : ffff8000101604c4: ret 0.28 : ffff8000101604c8: str x20, [x29, #24] : rnp = rdp->mynode; 0.82 : ffff8000101604cc: ldr x20, [x19, #24] : raw_spin_lock_rcu_node(rnp); 0.00 : ffff8000101604d0: mov x0, x20 0.00 : ffff8000101604d4: bl ffff800010cd80f0 <_raw_spin_lock> : t->rcu_read_unlock_special.b.blocked = true; 0.00 : ffff8000101604d8: mov w0, #0x1 // #1 0.00 : ffff8000101604dc: strb w0, [x21, #780] : t->rcu_blocked_node = rnp; 0.00 : ffff8000101604e0: str x20, [x21, #800] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101604e4: ldr x1, [x20, #56] : rcu_note_context_switch(): : WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0); 0.56 : ffff8000101604e8: ldr x0, [x19, #32] 0.00 : ffff8000101604ec: tst x1, x0 0.00 : ffff8000101604f0: b.eq ffff800010160758 // b.none : __read_once_size(): 0.00 : ffff8000101604f4: ldr x0, [x21, #784] : rcu_note_context_switch(): : WARN_ON_ONCE(!list_empty(&t->rcu_node_entry)); 0.00 : ffff8000101604f8: add x21, x21, #0x310 0.00 : ffff8000101604fc: cmp x21, x0 0.00 : ffff800010160500: b.ne ffff800010160750 // b.any : (rnp->qsmask & rdp->grpmask) 0.00 : ffff800010160504: ldr x3, [x19, #32] 0.00 : ffff800010160508: ldr x0, [x20, #32] : trace_rcu_preempt_task(rcu_state.name, 0.00 : ffff80001016050c: ands x1, x3, x0 0.00 : ffff800010160510: b.eq ffff800010160638 // b.none : rcu_preempt_ctxt_queue(): : (rnp->exp_tasks ? RCU_EXP_TASKS : 0) + 0.00 : ffff800010160514: ldp x0, x2, [x20, #144] : (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0); 0.00 : ffff800010160518: ldr x5, [x20, #64] : int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) + 0.00 : ffff80001016051c: cmp x0, #0x0 : WARN_ON_ONCE(rdp->mynode != rnp); 0.28 : ffff800010160520: ldr x4, [x19, #24] : int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) + 0.00 : ffff800010160524: cset w0, ne // ne = any 0.00 : ffff800010160528: cmp x2, #0x0 : get_current(): 0.00 : ffff80001016052c: mrs x2, sp_el0 : rcu_preempt_ctxt_queue(): 0.00 : ffff800010160530: lsl w0, w0, #3 0.00 : ffff800010160534: add w6, w0, #0x4 0.00 : ffff800010160538: csel w0, w6, w0, ne // ne = any 0.00 : ffff80001016053c: cmp x1, #0x0 0.00 : ffff800010160540: add w1, w0, #0x2 0.00 : ffff800010160544: csel w0, w1, w0, ne // ne = any : (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0); 0.00 : ffff800010160548: tst x3, x5 : int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) + 0.00 : ffff80001016054c: cinc w0, w0, ne // ne = any : WARN_ON_ONCE(rdp->mynode != rnp); 0.53 : ffff800010160550: cmp x20, x4 0.00 : ffff800010160554: b.ne ffff800010160740 // b.any : WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); 0.00 : ffff800010160558: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001016055c: ldrb w3, [x20, #113] 0.00 : ffff800010160560: ldr w1, [x1, #676] 0.00 : ffff800010160564: sub w1, w1, #0x1 0.00 : ffff800010160568: cmp w3, w1 0.00 : ffff80001016056c: b.ne ffff800010160748 // b.any : WARN_ON_ONCE(rnp->qsmaskinitnext & ~rnp->qsmaskinit & rnp->qsmask & 0.00 : ffff800010160570: ldp x3, x1, [x20, #48] 0.00 : ffff800010160574: ldr x5, [x19, #32] 0.00 : ffff800010160578: ldr x4, [x20, #32] 0.00 : ffff80001016057c: bic x3, x5, x3 0.00 : ffff800010160580: and x1, x1, x4 0.00 : ffff800010160584: tst x3, x1 0.00 : ffff800010160588: b.ne ffff80001016072c // b.any 0.00 : ffff80001016058c: add x1, x2, #0x310 : switch (blkd_state) { 0.00 : ffff800010160590: cmp w0, #0x9 0.00 : ffff800010160594: b.eq ffff8000101606cc // b.none 0.00 : ffff800010160598: b.le ffff800010160650 0.00 : ffff80001016059c: cmp w0, #0xd 0.00 : ffff8000101605a0: b.eq ffff8000101606e8 // b.none 0.00 : ffff8000101605a4: b.gt ffff8000101606bc 0.00 : ffff8000101605a8: cmp w0, #0xa 0.00 : ffff8000101605ac: b.ne ffff800010160708 // b.any : list_add(&t->rcu_node_entry, rnp->gp_tasks); 0.00 : ffff8000101605b0: ldr x3, [x20, #144] : list_add(): : * Insert a new entry after the specified head. : * This is good for implementing stacks. : */ : static inline void list_add(struct list_head *new, struct list_head *head) : { : __list_add(new, head, head->next); 0.00 : ffff8000101605b4: ldr x4, [x3] : __list_add(): : next->prev = new; 0.00 : ffff8000101605b8: str x1, [x4, #8] : new->next = next; 0.00 : ffff8000101605bc: str x4, [x2, #784] : new->prev = prev; 0.00 : ffff8000101605c0: str x3, [x2, #792] : __write_once_size(): : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101605c4: str x1, [x3] : rcu_preempt_ctxt_queue(): : if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) { 0.00 : ffff8000101605c8: ldr x3, [x20, #144] 0.00 : ffff8000101605cc: and w2, w0, #0x2 0.00 : ffff8000101605d0: cbz x3, ffff80001016069c : if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD)) 0.00 : ffff8000101605d4: ldr x3, [x20, #152] 0.00 : ffff8000101605d8: and w21, w0, #0x1 0.00 : ffff8000101605dc: cbz x3, ffff800010160690 : WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) != 0.28 : ffff8000101605e0: ldr x1, [x19, #32] 0.00 : ffff8000101605e4: cmp w2, #0x0 0.00 : ffff8000101605e8: ldr x4, [x20, #32] 0.00 : ffff8000101605ec: cset w3, eq // eq = none 0.00 : ffff8000101605f0: tst x1, x4 0.00 : ffff8000101605f4: cset w2, eq // eq = none 0.29 : ffff8000101605f8: cmp w3, w2 0.00 : ffff8000101605fc: b.ne ffff800010160734 // b.any : WARN_ON_ONCE(!(blkd_state & RCU_EXP_BLKD) != 0.00 : ffff800010160600: ldr x2, [x20, #64] 0.00 : ffff800010160604: mvn w0, w0 0.00 : ffff800010160608: and w0, w0, #0x1 0.00 : ffff80001016060c: tst x1, x2 0.00 : ffff800010160610: cset w1, eq // eq = none 0.00 : ffff800010160614: cmp w1, w0 0.00 : ffff800010160618: b.ne ffff800010160724 // b.any : raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */ 0.00 : ffff80001016061c: mov x0, x20 0.00 : ffff800010160620: bl ffff800010cd7c38 <_raw_spin_unlock> : if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs) 0.00 : ffff800010160624: ldrb w0, [x19, #21] 0.00 : ffff800010160628: cbz w21, ffff800010160680 0.00 : ffff80001016062c: cbnz w0, ffff8000101606f0 0.00 : ffff800010160630: ldr x20, [x29, #24] 0.00 : ffff800010160634: b ffff80001016049c : __read_once_size(): : __READ_ONCE_SIZE; 0.28 : ffff800010160638: ldr x0, [x20, #8] : rcu_seq_snap(): : static inline unsigned long rcu_seq_snap(unsigned long *sp) : { : unsigned long s; : : s = (READ_ONCE(*sp) + 2 * RCU_SEQ_STATE_MASK + 1) & ~RCU_SEQ_STATE_MASK; : smp_mb(); /* Above access must not bleed into critical section. */ 0.00 : ffff80001016063c: dmb ish 0.28 : ffff800010160640: ldr x3, [x19, #32] 0.00 : ffff800010160644: ldr x0, [x20, #32] 0.00 : ffff800010160648: and x1, x3, x0 0.00 : ffff80001016064c: b ffff800010160514 : rcu_preempt_ctxt_queue(): : switch (blkd_state) { 0.00 : ffff800010160650: cmp w0, #0x5 0.00 : ffff800010160654: b.eq ffff8000101606e8 // b.none 0.00 : ffff800010160658: b.le ffff800010160714 0.00 : ffff80001016065c: cmp w0, #0x7 0.00 : ffff800010160660: b.eq ffff8000101606e8 // b.none : list_add(): : __list_add(new, head, head->next); 0.00 : ffff800010160664: mov x3, x20 0.00 : ffff800010160668: ldr x4, [x3, #128]! : __list_add(): : next->prev = new; 0.00 : ffff80001016066c: str x1, [x4, #8] : new->next = next; 0.28 : ffff800010160670: str x4, [x2, #784] : new->prev = prev; 0.00 : ffff800010160674: str x3, [x2, #792] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010160678: str x1, [x20, #128] 0.00 : ffff80001016067c: b ffff8000101605c8 : rcu_preempt_ctxt_queue(): : WARN_ON_ONCE(rdp->exp_deferred_qs); 0.00 : ffff800010160680: cbz w0, ffff800010160630 0.00 : ffff800010160684: brk #0x800 0.00 : ffff800010160688: ldr x20, [x29, #24] 0.00 : ffff80001016068c: b ffff80001016049c : if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD)) 0.00 : ffff800010160690: cbz w21, ffff8000101605e0 : rnp->exp_tasks = &t->rcu_node_entry; 0.00 : ffff800010160694: str x1, [x20, #152] 0.00 : ffff800010160698: b ffff8000101605e0 : if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) { 0.28 : ffff80001016069c: cbz w2, ffff8000101605d4 : WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq); 0.00 : ffff8000101606a0: ldr x3, [x20, #8] 0.00 : ffff8000101606a4: ldr x4, [x20, #24] : __write_once_size(): 0.00 : ffff8000101606a8: str x1, [x20, #144] : rcu_preempt_ctxt_queue(): 0.00 : ffff8000101606ac: cmp x4, x3 0.00 : ffff8000101606b0: b.ne ffff8000101605d4 // b.any 0.00 : ffff8000101606b4: brk #0x800 0.00 : ffff8000101606b8: b ffff8000101605d4 : switch (blkd_state) { 0.00 : ffff8000101606bc: cmp w0, #0xe 0.00 : ffff8000101606c0: b.eq ffff8000101605b0 // b.none 0.00 : ffff8000101606c4: cmp w0, #0xf 0.00 : ffff8000101606c8: b.ne ffff800010160664 // b.any : list_add_tail(): : * Insert a new entry before the specified head. : * This is useful for implementing queues. : */ : static inline void list_add_tail(struct list_head *new, struct list_head *head) : { : __list_add(new, head->prev, head); 0.28 : ffff8000101606cc: ldr x3, [x20, #136] : rcu_preempt_ctxt_queue(): : list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks); 0.00 : ffff8000101606d0: add x4, x20, #0x80 : __list_add(): : next->prev = new; 0.00 : ffff8000101606d4: str x1, [x20, #136] : new->next = next; 0.00 : ffff8000101606d8: str x4, [x2, #784] : new->prev = prev; 0.00 : ffff8000101606dc: str x3, [x2, #792] : __write_once_size(): 0.00 : ffff8000101606e0: str x1, [x3] 0.00 : ffff8000101606e4: b ffff8000101605c8 : rcu_preempt_ctxt_queue(): : list_add(&t->rcu_node_entry, rnp->exp_tasks); 0.00 : ffff8000101606e8: ldr x3, [x20, #152] 0.00 : ffff8000101606ec: b ffff8000101605b4 : rcu_report_exp_rdp(): 0.00 : ffff8000101606f0: ldp x0, x1, [x19, #24] : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff8000101606f4: strb wzr, [x19, #21] : rcu_report_exp_rdp(): 0.00 : ffff8000101606f8: mov w2, #0x1 // #1 0.00 : ffff8000101606fc: bl ffff80001015c340 0.00 : ffff800010160700: ldr x20, [x29, #24] 0.00 : ffff800010160704: b ffff80001016049c : rcu_preempt_ctxt_queue(): : switch (blkd_state) { 0.00 : ffff800010160708: cmp w0, #0xb 0.00 : ffff80001016070c: b.eq ffff8000101606cc // b.none 0.00 : ffff800010160710: b ffff800010160664 0.28 : ffff800010160714: sub w3, w0, #0x1 0.00 : ffff800010160718: cmp w3, #0x2 0.00 : ffff80001016071c: b.hi ffff800010160664 // b.pmore 0.00 : ffff800010160720: b ffff8000101606cc : WARN_ON_ONCE(!(blkd_state & RCU_EXP_BLKD) != 0.00 : ffff800010160724: brk #0x800 0.00 : ffff800010160728: b ffff80001016061c : WARN_ON_ONCE(rnp->qsmaskinitnext & ~rnp->qsmaskinit & rnp->qsmask & 0.00 : ffff80001016072c: brk #0x800 0.00 : ffff800010160730: b ffff80001016058c : WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) != 0.00 : ffff800010160734: brk #0x800 0.00 : ffff800010160738: ldr x1, [x19, #32] 0.00 : ffff80001016073c: b ffff800010160600 : WARN_ON_ONCE(rdp->mynode != rnp); 0.00 : ffff800010160740: brk #0x800 0.00 : ffff800010160744: b ffff800010160558 : WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); 0.00 : ffff800010160748: brk #0x800 0.00 : ffff80001016074c: b ffff800010160570 : rcu_note_context_switch(): : WARN_ON_ONCE(!list_empty(&t->rcu_node_entry)); 0.00 : ffff800010160750: brk #0x800 0.00 : ffff800010160754: b ffff800010160504 : WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0); 0.00 : ffff800010160758: brk #0x800 0.00 : ffff80001016075c: b ffff8000101604f4 Percent | Source code & Disassembly of vmlinux for cycles (491 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010205408 : : gup_pgd_range(): : return 1; : } : : static void gup_pgd_range(unsigned long addr, unsigned long end, : unsigned int flags, struct page **pages, int *nr) : { 0.77 : ffff800010205408: stp x29, x30, [sp, #-176]! 0.00 : ffff80001020540c: mov x6, x1 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 4.72 : ffff800010205410: mrs x5, sp_el0 : gup_pgd_range(): 0.00 : ffff800010205414: mov x29, sp 0.00 : ffff800010205418: stp x19, x20, [sp, #16] : unsigned long next; : pgd_t *pgdp; : : pgdp = pgd_offset(current->mm, addr); 0.00 : ffff80001020541c: ubfx x1, x0, #39, #9 : { 0.21 : ffff800010205420: stp x21, x22, [sp, #32] : gup_pte_range(): : if (!pte_access_permitted(pte, flags & FOLL_WRITE)) 0.00 : ffff800010205424: mov x22, #0x41 // #65 : gup_pgd_range(): : { 0.41 : ffff800010205428: stp x23, x24, [sp, #48] 0.00 : ffff80001020542c: mov x24, x0 0.00 : ffff800010205430: stp x25, x26, [sp, #64] 0.00 : ffff800010205434: mov x26, x4 0.20 : ffff800010205438: stp x27, x28, [sp, #80] : gup_pud_range(): : if (unlikely(!pud_present(pud))) 0.00 : ffff80001020543c: mov x27, #0x1 // #1 : gup_pgd_range(): : pgdp = pgd_offset(current->mm, addr); 0.00 : ffff800010205440: mov x28, x3 : { 0.40 : ffff800010205444: str w2, [x29, #160] : pgdp = pgd_offset(current->mm, addr); 0.00 : ffff800010205448: ldr x5, [x5, #952] 0.00 : ffff80001020544c: sub x2, x6, #0x1 : gup_pud_range(): : if (unlikely(!pud_present(pud))) 0.00 : ffff800010205450: movk x27, #0x400, lsl #48 0.39 : ffff800010205454: str x2, [x29, #96] : gup_pgd_range(): : pgdp = pgd_offset(current->mm, addr); 0.00 : ffff800010205458: ldr x0, [x5, #64] 0.00 : ffff80001020545c: add x0, x0, x1, lsl #3 4.48 : ffff800010205460: stp x6, x0, [x29, #104] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.87 : ffff800010205464: ldr x0, [x29, #112] : gup_pgd_range(): : do { : pgd_t pgd = READ_ONCE(*pgdp); : : next = pgd_addr_end(addr, end); 0.00 : ffff800010205468: mov x20, #0x8000000000 // #549755813888 0.00 : ffff80001020546c: add x20, x24, x20 6.77 : ffff800010205470: ldr x2, [x29, #96] 0.00 : ffff800010205474: and x20, x20, #0xffffff8000000000 : __read_once_size(): 0.21 : ffff800010205478: ldr x0, [x0] : gup_pgd_range(): 0.00 : ffff80001020547c: sub x1, x20, #0x1 0.00 : ffff800010205480: cmp x1, x2 5.29 : ffff800010205484: ldr x1, [x29, #104] 0.00 : ffff800010205488: csel x20, x20, x1, cc // cc = lo, ul, last : if (pgd_none(pgd)) 0.00 : ffff80001020548c: cbz x0, ffff80001020557c : gup_pud_range(): : pudp = pud_offset(&p4d, addr); 0.00 : ffff800010205490: adrp x25, ffff8000112f2000 0.00 : ffff800010205494: sub x1, x20, #0x1 0.21 : ffff800010205498: str x1, [x29, #128] 0.00 : ffff80001020549c: ubfx x19, x24, #30, #9 0.00 : ffff8000102054a0: ldr x1, [x25, #1872] : pgd_page_paddr(): : set_pgd(pgdp, __pgd(0)); : } : : static inline phys_addr_t pgd_page_paddr(pgd_t pgd) : { : return __pgd_to_phys(pgd); 0.00 : ffff8000102054a4: and x0, x0, #0xfffffffff000 : gup_pud_range(): 0.00 : ffff8000102054a8: mov x21, x28 0.00 : ffff8000102054ac: str x20, [x29, #168] 0.00 : ffff8000102054b0: sub x0, x0, x1 0.00 : ffff8000102054b4: add x19, x0, x19, lsl #3 : next = pud_addr_end(addr, end); 0.00 : ffff8000102054b8: mov x11, #0x40000000 // #1073741824 0.00 : ffff8000102054bc: add x11, x24, x11 0.00 : ffff8000102054c0: and x20, x11, #0xffffffffc0000000 : __read_once_size(): 0.62 : ffff8000102054c4: ldr x23, [x19] : gup_pud_range(): 5.34 : ffff8000102054c8: ldr x1, [x29, #128] 0.00 : ffff8000102054cc: sub x0, x20, #0x1 0.00 : ffff8000102054d0: cmp x0, x1 0.00 : ffff8000102054d4: ldr x0, [x29, #168] 0.00 : ffff8000102054d8: csel x20, x20, x0, cc // cc = lo, ul, last : if (unlikely(!pud_present(pud))) 0.00 : ffff8000102054dc: tst x23, x27 0.00 : ffff8000102054e0: b.eq ffff80001020557c // b.none : if (unlikely(pud_huge(pud))) { 0.00 : ffff8000102054e4: mov x0, x23 0.00 : ffff8000102054e8: bl ffff8000100a5400 0.00 : ffff8000102054ec: cbnz w0, ffff8000102058f8 : gup_pmd_range(): : pmdp = pmd_offset(&pud, addr); 0.00 : ffff8000102054f0: ldr x1, [x25, #1872] 0.00 : ffff8000102054f4: ubfx x7, x24, #21, #9 : pud_page_paddr(): : return __pud_to_phys(pud); 0.00 : ffff8000102054f8: and x0, x23, #0xfffffffff000 0.00 : ffff8000102054fc: sub x12, x20, #0x1 : gup_pmd_range(): 1.42 : ffff800010205500: sub x0, x0, x1 : gup_pte_range(): : if (!pte_access_permitted(pte, flags & FOLL_WRITE)) 0.00 : ffff800010205504: ldr w1, [x29, #160] : gup_pmd_range(): : pmdp = pmd_offset(&pud, addr); 0.00 : ffff800010205508: add x7, x0, x7, lsl #3 0.00 : ffff80001020550c: str x19, [x29, #120] : gup_pte_range(): : if (!pte_access_permitted(pte, flags & FOLL_WRITE)) 0.00 : ffff800010205510: and w28, w1, #0x1 : gup_pmd_range(): : pmdp = pmd_offset(&pud, addr); 0.00 : ffff800010205514: mov x19, x20 0.00 : ffff800010205518: mov x20, x7 0.00 : ffff80001020551c: nop : next = pmd_addr_end(addr, end); 0.60 : ffff800010205520: add x23, x24, #0x200, lsl #12 : __read_once_size(): 0.00 : ffff800010205524: ldr x3, [x20] : gup_pmd_range(): 0.00 : ffff800010205528: and x23, x23, #0xffffffffffe00000 0.00 : ffff80001020552c: sub x0, x23, #0x1 0.00 : ffff800010205530: cmp x0, x12 6.32 : ffff800010205534: csel x23, x23, x19, cc // cc = lo, ul, last : if (!pmd_present(pmd)) 0.00 : ffff800010205538: ands x1, x3, x27 0.00 : ffff80001020553c: b.eq ffff80001020557c // b.none : if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) || 0.00 : ffff800010205540: tbz w3, #1, ffff800010205564 0.21 : ffff800010205544: mov x0, x3 0.21 : ffff800010205548: stp x12, x1, [x29, #136] 0.00 : ffff80001020554c: str x3, [x29, #152] 0.00 : ffff800010205550: bl ffff8000100a53e8 0.00 : ffff800010205554: ldp x12, x1, [x29, #136] 1.83 : ffff800010205558: ldr x3, [x29, #152] 0.00 : ffff80001020555c: cbnz w0, ffff800010205564 0.00 : ffff800010205560: tbz x3, #57, ffff800010205598 : if (pmd_protnone(pmd)) 0.00 : ffff800010205564: mov x0, #0x400000000000000 // #288230376151711744 0.00 : ffff800010205568: cmp x1, x0 0.00 : ffff80001020556c: b.eq ffff80001020557c // b.none : gup_huge_pmd(): : if (!pmd_access_permitted(orig, flags & FOLL_WRITE)) 0.00 : ffff800010205570: and x0, x3, x22 0.00 : ffff800010205574: cmp x0, #0x41 0.00 : ffff800010205578: b.eq ffff8000102057a4 // b.none : gup_pgd_range(): : PGDIR_SHIFT, next, flags, pages, nr)) : return; : } else if (!gup_p4d_range(pgd, addr, next, flags, pages, nr)) : return; : } while (pgdp++, addr = next, addr != end); : } 0.00 : ffff80001020557c: ldp x19, x20, [sp, #16] 0.00 : ffff800010205580: ldp x21, x22, [sp, #32] 0.00 : ffff800010205584: ldp x23, x24, [sp, #48] 0.00 : ffff800010205588: ldp x25, x26, [sp, #64] 0.00 : ffff80001020558c: ldp x27, x28, [sp, #80] 0.00 : ffff800010205590: ldp x29, x30, [sp], #176 0.00 : ffff800010205594: ret : gup_pte_range(): : ptem = ptep = pte_offset_map(&pmd, addr); 0.00 : ffff800010205598: ldr x0, [x25, #1872] 0.00 : ffff80001020559c: ubfx x5, x24, #12, #9 : pmd_page_paddr(): : return __pmd_to_phys(pmd); 0.00 : ffff8000102055a0: and x3, x3, #0xfffffffff000 : gup_pte_range(): : int nr_start = *nr, ret = 0; 0.00 : ffff8000102055a4: ldr w18, [x26] : ptem = ptep = pte_offset_map(&pmd, addr); 0.00 : ffff8000102055a8: sub x3, x3, x0 : if (pte_protnone(pte)) 0.00 : ffff8000102055ac: mov x10, #0x400000000000000 // #288230376151711744 : ptem = ptep = pte_offset_map(&pmd, addr); 0.00 : ffff8000102055b0: add x5, x3, x5, lsl #3 : page = pte_page(pte); 0.41 : ffff8000102055b4: adrp x11, ffff8000112f2000 : __lse_atomic64_or(): : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) 0.00 : ffff8000102055b8: mov x13, #0x2 // #2 : __read_once_size(): 0.00 : ffff8000102055bc: ldr x3, [x5] : pte_protnone(): : return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE; 0.82 : ffff8000102055c0: and x0, x3, x27 : gup_pte_range(): : if (pte_protnone(pte)) 0.00 : ffff8000102055c4: cmp x0, x10 0.00 : ffff8000102055c8: b.eq ffff80001020557c // b.none : if (!pte_access_permitted(pte, flags & FOLL_WRITE)) 5.46 : ffff8000102055cc: and x0, x3, x22 0.00 : ffff8000102055d0: cmp x0, #0x41 0.20 : ffff8000102055d4: b.ne ffff80001020557c // b.any 1.23 : ffff8000102055d8: cbz w28, ffff8000102055e0 0.00 : ffff8000102055dc: tbz x3, #51, ffff80001020557c : if (pte_devmap(pte)) { 0.00 : ffff8000102055e0: tbnz x3, #57, ffff800010205788 : } else if (pte_special(pte)) 0.21 : ffff8000102055e4: tbnz x3, #56, ffff80001020557c : page = pte_page(pte); 0.00 : ffff8000102055e8: ldr x0, [x11, #1880] 0.00 : ffff8000102055ec: ubfx x4, x3, #12, #36 0.00 : ffff8000102055f0: add x4, x0, x4, lsl #6 : __read_once_size(): 0.00 : ffff8000102055f4: ldr x0, [x4, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff8000102055f8: sub x9, x0, #0x1 0.00 : ffff8000102055fc: tst x0, #0x1 0.00 : ffff800010205600: csel x9, x9, x4, ne // ne = any : __read_once_size(): 11.38 : ffff800010205604: ldr w0, [x9, #52] : try_get_compound_head(): : if (WARN_ON_ONCE(page_ref_count(head) < 0)) 0.00 : ffff800010205608: tbnz w0, #31, ffff800010205734 : __read_once_size(): 3.29 : ffff80001020560c: ldr w6, [x9, #52] : atomic_fetch_add_unless(): : atomic_fetch_add_unless(atomic_t *v, int a, int u) : { : int c = atomic_read(v); : : do { : if (unlikely(c == u)) 0.00 : ffff800010205610: cbz w6, ffff80001020557c 0.19 : ffff800010205614: add x8, x9, #0x34 : atomic_cmpxchg(): : #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) : static inline int : atomic_cmpxchg(atomic_t *v, int old, int new) : { : kasan_check_write(v, sizeof(*v)); : return arch_atomic_cmpxchg(v, old, new); 0.00 : ffff800010205618: sxtw x1, w6 : atomic_fetch_add_unless(): : break; : } while (!atomic_try_cmpxchg(v, &c, c + a)); 0.00 : ffff80001020561c: add w2, w6, #0x1 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010205620: b ffff8000102056f8 0.21 : ffff800010205624: b ffff8000102056f8 : __lse__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 0.00 : ffff800010205628: mov x0, x8 0.00 : ffff80001020562c: mov w7, w1 0.00 : ffff800010205630: casal w7, w2, [x8] 14.42 : ffff800010205634: mov w0, w7 : atomic_try_cmpxchg(): : if (unlikely(r != o)) 0.00 : ffff800010205638: cmp w0, w6 0.00 : ffff80001020563c: b.ne ffff800010205708 // b.any : gup_pte_range(): : if (!head) 0.00 : ffff800010205640: cbz x9, ffff80001020557c : if (unlikely(pte_val(pte) != pte_val(*ptep))) { 0.20 : ffff800010205644: ldr x0, [x5] 0.00 : ffff800010205648: cmp x3, x0 0.00 : ffff80001020564c: b.ne ffff80001020573c // b.any : __read_once_size(): 5.90 : ffff800010205650: ldr x1, [x4, #8] : compound_head(): 0.00 : ffff800010205654: sub x0, x1, #0x1 0.00 : ffff800010205658: tst x1, #0x1 0.00 : ffff80001020565c: csel x0, x0, x4, ne // ne = any : arch_static_branch_jump(): 5.48 : ffff800010205660: b ffff80001020572c 0.00 : ffff800010205664: b ffff80001020572c : __lse_atomic64_or(): : ATOMIC64_OP(or, stset) 0.00 : ffff800010205668: mov x1, x13 0.00 : ffff80001020566c: stset x1, [x0] : gup_pte_range(): : pages[*nr] = page; 4.50 : ffff800010205670: ldrsw x0, [x26] : } while (ptep++, addr += PAGE_SIZE, addr != end); 0.00 : ffff800010205674: add x24, x24, #0x1, lsl #12 0.00 : ffff800010205678: add x5, x5, #0x8 0.00 : ffff80001020567c: cmp x23, x24 : pages[*nr] = page; 0.00 : ffff800010205680: str x4, [x21, x0, lsl #3] : (*nr)++; 0.00 : ffff800010205684: ldr w0, [x26] 0.00 : ffff800010205688: add w0, w0, #0x1 0.00 : ffff80001020568c: str w0, [x26] : } while (ptep++, addr += PAGE_SIZE, addr != end); 0.00 : ffff800010205690: b.ne ffff8000102055bc // b.any 0.19 : ffff800010205694: mov x24, x23 : gup_pmd_range(): : } while (pmdp++, addr = next, addr != end); 0.00 : ffff800010205698: add x20, x20, #0x8 0.00 : ffff80001020569c: cmp x19, x23 0.00 : ffff8000102056a0: b.ne ffff800010205520 // b.any 0.00 : ffff8000102056a4: mov x20, x19 0.00 : ffff8000102056a8: ldr x19, [x29, #120] : gup_pud_range(): : } while (pudp++, addr = next, addr != end); 0.00 : ffff8000102056ac: ldr x0, [x29, #168] 0.00 : ffff8000102056b0: mov x24, x20 0.00 : ffff8000102056b4: add x19, x19, #0x8 0.21 : ffff8000102056b8: cmp x0, x20 0.81 : ffff8000102056bc: b.ne ffff8000102054b8 // b.any : gup_pgd_range(): : } while (pgdp++, addr = next, addr != end); 0.00 : ffff8000102056c0: ldr x0, [x29, #112] 0.00 : ffff8000102056c4: mov x28, x21 0.00 : ffff8000102056c8: add x0, x0, #0x8 0.21 : ffff8000102056cc: str x0, [x29, #112] 0.00 : ffff8000102056d0: ldr x0, [x29, #104] 0.00 : ffff8000102056d4: cmp x20, x0 0.00 : ffff8000102056d8: b.ne ffff800010205464 // b.any : } 0.41 : ffff8000102056dc: ldp x19, x20, [sp, #16] 0.00 : ffff8000102056e0: ldp x21, x22, [sp, #32] 0.00 : ffff8000102056e4: ldp x23, x24, [sp, #48] 0.41 : ffff8000102056e8: ldp x25, x26, [sp, #64] 0.20 : ffff8000102056ec: ldp x27, x28, [sp, #80] 0.19 : ffff8000102056f0: ldp x29, x30, [sp], #176 0.00 : ffff8000102056f4: ret : __ll_sc__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 0.00 : ffff8000102056f8: and x1, x1, #0xffffffff 0.00 : ffff8000102056fc: b ffff800010207be8 : atomic_try_cmpxchg(): 0.00 : ffff800010205700: cmp w0, w6 0.00 : ffff800010205704: b.eq ffff800010205640 // b.none 0.00 : ffff800010205708: mov w6, w0 : atomic_fetch_add_unless(): : if (unlikely(c == u)) 0.00 : ffff80001020570c: cbnz w0, ffff800010205618 : gup_pgd_range(): 0.00 : ffff800010205710: ldp x19, x20, [sp, #16] 0.00 : ffff800010205714: ldp x21, x22, [sp, #32] 0.00 : ffff800010205718: ldp x23, x24, [sp, #48] 0.00 : ffff80001020571c: ldp x25, x26, [sp, #64] 0.00 : ffff800010205720: ldp x27, x28, [sp, #80] 0.00 : ffff800010205724: ldp x29, x30, [sp], #176 0.00 : ffff800010205728: ret : __ll_sc_atomic64_or(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff80001020572c: b ffff800010207c08 0.00 : ffff800010205730: b ffff800010205670 : try_get_compound_head(): : if (WARN_ON_ONCE(page_ref_count(head) < 0)) 0.00 : ffff800010205734: brk #0x800 0.00 : ffff800010205738: b ffff80001020557c : __read_once_size(): 0.00 : ffff80001020573c: ldr x0, [x9, #8] : compound_head(): : if (unlikely(head & 1)) 0.00 : ffff800010205740: tbnz w0, #0, ffff80001020576c : arch_static_branch_jump(): 0.00 : ffff800010205744: b ffff800010205778 0.00 : ffff800010205748: b ffff800010205778 : __lse_atomic_sub_return(): : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001020574c: mov w0, #0x1 // #1 0.00 : ffff800010205750: neg w0, w0 0.00 : ffff800010205754: ldaddal w0, w1, [x8] 0.00 : ffff800010205758: add w0, w0, w1 : put_page(): : if (page_is_devmap_managed(page)) { : put_devmap_managed_page(page); : return; : } : : if (put_page_testzero(page)) 0.00 : ffff80001020575c: cbnz w0, ffff80001020557c : __put_page(page); 0.00 : ffff800010205760: mov x0, x9 0.00 : ffff800010205764: bl ffff8000101dfec0 <__put_page> 0.00 : ffff800010205768: b ffff80001020557c : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff80001020576c: sub x9, x0, #0x1 0.00 : ffff800010205770: add x8, x0, #0x33 0.00 : ffff800010205774: b ffff800010205744 : __ll_sc_atomic_sub_return(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010205778: mov w1, #0x1 // #1 0.00 : ffff80001020577c: b ffff800010207c20 : put_page(): : if (put_page_testzero(page)) 0.00 : ffff800010205780: cbnz w0, ffff80001020557c 0.00 : ffff800010205784: b ffff800010205760 : gup_pte_range(): : if (unlikely(flags & FOLL_LONGTERM)) 0.00 : ffff800010205788: ldr x0, [x29, #160] 0.00 : ffff80001020578c: tbnz w0, #16, ffff80001020557c : undo_dev_pagemap(nr, nr_start, pages); 0.00 : ffff800010205790: mov x2, x21 0.00 : ffff800010205794: mov w1, w18 0.00 : ffff800010205798: mov x0, x26 0.00 : ffff80001020579c: bl ffff800010205190 0.00 : ffff8000102057a0: b ffff80001020557c : gup_huge_pmd(): : if (!pmd_access_permitted(orig, flags & FOLL_WRITE)) 0.00 : ffff8000102057a4: cbz w28, ffff8000102057ac 0.00 : ffff8000102057a8: tbz x3, #51, ffff80001020557c : if (pmd_devmap(orig)) { 0.00 : ffff8000102057ac: tbz x3, #57, ffff8000102057cc : if (unlikely(flags & FOLL_LONGTERM)) 0.00 : ffff8000102057b0: ldr x0, [x29, #160] 0.00 : ffff8000102057b4: tbnz w0, #16, ffff80001020557c : __gup_device_huge(): : undo_dev_pagemap(nr, nr_start, pages); 0.00 : ffff8000102057b8: ldr w1, [x26] 0.00 : ffff8000102057bc: mov x2, x21 0.00 : ffff8000102057c0: mov x0, x26 0.00 : ffff8000102057c4: bl ffff800010205190 0.00 : ffff8000102057c8: b ffff80001020557c : gup_huge_pmd(): : page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 0.00 : ffff8000102057cc: adrp x8, ffff8000112f2000 0.00 : ffff8000102057d0: ubfx x5, x3, #12, #36 0.00 : ffff8000102057d4: ubfx x0, x24, #12, #9 : record_subpages(): : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff8000102057d8: cmp x24, x23 : gup_huge_pmd(): : page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 0.00 : ffff8000102057dc: ldr x1, [x8, #1880] 0.00 : ffff8000102057e0: add x0, x0, x5 : refs = record_subpages(page, addr, end, pages + *nr); 0.00 : ffff8000102057e4: ldr w4, [x26] : page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 0.00 : ffff8000102057e8: add x0, x1, x0, lsl #6 : record_subpages(): : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff8000102057ec: b.eq ffff800010205a18 // b.none 0.00 : ffff8000102057f0: mov w2, #0x0 // #0 0.00 : ffff8000102057f4: nop : pages[nr++] = page++; 0.00 : ffff8000102057f8: sbfiz x1, x2, #3, #32 0.00 : ffff8000102057fc: add w6, w2, #0x1 0.00 : ffff800010205800: add x1, x1, w4, sxtw #3 : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff800010205804: add x24, x24, #0x1, lsl #12 : pages[nr++] = page++; 0.00 : ffff800010205808: mov w2, w6 : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff80001020580c: cmp x23, x24 : pages[nr++] = page++; 0.00 : ffff800010205810: str x0, [x21, x1] 0.00 : ffff800010205814: add x0, x0, #0x40 : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff800010205818: b.ne ffff8000102057f8 // b.any 0.00 : ffff80001020581c: ldr x1, [x8, #1880] : gup_huge_pmd(): : head = try_get_compound_head(pmd_page(orig), refs); 0.00 : ffff800010205820: add x5, x1, x5, lsl #6 : __read_once_size(): 0.00 : ffff800010205824: ldr x0, [x5, #8] : compound_head(): 0.00 : ffff800010205828: sub x1, x0, #0x1 0.00 : ffff80001020582c: tst x0, #0x1 0.00 : ffff800010205830: csel x5, x1, x5, ne // ne = any : __read_once_size(): 0.00 : ffff800010205834: ldr w0, [x5, #52] : try_get_compound_head(): : if (WARN_ON_ONCE(page_ref_count(head) < 0)) 0.00 : ffff800010205838: tbnz w0, #31, ffff800010205734 : __read_once_size(): 0.00 : ffff80001020583c: ldr w8, [x5, #52] : atomic_fetch_add_unless(): 0.00 : ffff800010205840: cbz w8, ffff80001020557c 0.00 : ffff800010205844: add x4, x5, #0x34 : atomic_cmpxchg(): 0.00 : ffff800010205848: sxtw x1, w8 : atomic_fetch_add_unless(): : } while (!atomic_try_cmpxchg(v, &c, c + a)); 0.00 : ffff80001020584c: add w2, w8, w6 : arch_static_branch_jump(): 0.00 : ffff800010205850: b ffff8000102058c0 0.00 : ffff800010205854: b ffff8000102058c0 : __lse__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 0.00 : ffff800010205858: mov x0, x4 0.00 : ffff80001020585c: mov w7, w1 0.00 : ffff800010205860: casal w7, w2, [x4] 0.00 : ffff800010205864: mov w0, w7 : atomic_try_cmpxchg(): : if (unlikely(r != o)) 0.00 : ffff800010205868: cmp w0, w8 0.00 : ffff80001020586c: b.ne ffff8000102058d4 // b.any : gup_huge_pmd(): : if (!head) 0.00 : ffff800010205870: cbz x5, ffff80001020557c : if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { 0.00 : ffff800010205874: ldr x0, [x20] 0.00 : ffff800010205878: cmp x3, x0 0.00 : ffff80001020587c: b.ne ffff800010205a20 // b.any : *nr += refs; 0.00 : ffff800010205880: ldr w0, [x26] 0.00 : ffff800010205884: add w6, w0, w6 0.00 : ffff800010205888: str w6, [x26] : __read_once_size(): 0.00 : ffff80001020588c: ldr x0, [x5, #8] : compound_head(): 0.00 : ffff800010205890: sub x1, x0, #0x1 0.00 : ffff800010205894: tst x0, #0x1 0.00 : ffff800010205898: csel x5, x1, x5, ne // ne = any : arch_static_branch_jump(): 0.00 : ffff80001020589c: b ffff8000102058e0 0.00 : ffff8000102058a0: b ffff8000102058e0 : __lse_atomic64_or(): : ATOMIC64_OP(or, stset) 0.00 : ffff8000102058a4: mov x0, #0x2 // #2 0.00 : ffff8000102058a8: stset x0, [x5] 0.00 : ffff8000102058ac: mov x24, x23 : gup_pmd_range(): : } while (pmdp++, addr = next, addr != end); 0.00 : ffff8000102058b0: add x20, x20, #0x8 0.00 : ffff8000102058b4: cmp x19, x23 0.00 : ffff8000102058b8: b.ne ffff800010205520 // b.any 0.00 : ffff8000102058bc: b ffff8000102056a4 : __ll_sc__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 0.00 : ffff8000102058c0: and x0, x1, #0xffffffff 0.00 : ffff8000102058c4: b ffff800010207c3c 0.00 : ffff8000102058c8: mov w0, w1 : atomic_try_cmpxchg(): 0.00 : ffff8000102058cc: cmp w0, w8 0.00 : ffff8000102058d0: b.eq ffff800010205870 // b.none 0.00 : ffff8000102058d4: mov w8, w0 : atomic_fetch_add_unless(): : if (unlikely(c == u)) 0.00 : ffff8000102058d8: cbnz w0, ffff800010205848 0.00 : ffff8000102058dc: b ffff80001020557c : __ll_sc_atomic64_or(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000102058e0: b ffff800010207c5c 0.00 : ffff8000102058e4: mov x24, x23 : gup_pmd_range(): 0.00 : ffff8000102058e8: add x20, x20, #0x8 0.00 : ffff8000102058ec: cmp x19, x23 0.00 : ffff8000102058f0: b.ne ffff800010205520 // b.any 0.00 : ffff8000102058f4: b ffff8000102056a4 : gup_huge_pud(): : if (!pud_access_permitted(orig, flags & FOLL_WRITE)) 0.00 : ffff8000102058f8: and x0, x23, x22 0.00 : ffff8000102058fc: cmp x0, #0x41 0.00 : ffff800010205900: b.ne ffff80001020557c // b.any 0.00 : ffff800010205904: ldr x0, [x29, #160] 0.00 : ffff800010205908: tbz w0, #0, ffff800010205910 0.00 : ffff80001020590c: tbz x23, #51, ffff80001020557c : page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 0.00 : ffff800010205910: adrp x6, ffff8000112f2000 0.00 : ffff800010205914: ubfx x4, x23, #12, #36 0.00 : ffff800010205918: ubfx x0, x24, #12, #18 : record_subpages(): : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff80001020591c: cmp x24, x20 : gup_huge_pud(): : page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 0.00 : ffff800010205920: ldr x1, [x6, #1880] 0.00 : ffff800010205924: add x0, x0, x4 : refs = record_subpages(page, addr, end, pages + *nr); 0.00 : ffff800010205928: ldr w3, [x26] : page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 0.00 : ffff80001020592c: add x0, x1, x0, lsl #6 : record_subpages(): : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff800010205930: b.eq ffff800010205a30 // b.none 0.00 : ffff800010205934: mov w2, #0x0 // #0 : pages[nr++] = page++; 0.00 : ffff800010205938: sbfiz x1, x2, #3, #32 0.00 : ffff80001020593c: add w5, w2, #0x1 0.00 : ffff800010205940: add x1, x1, w3, sxtw #3 : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff800010205944: add x24, x24, #0x1, lsl #12 : pages[nr++] = page++; 0.00 : ffff800010205948: mov w2, w5 : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff80001020594c: cmp x20, x24 : pages[nr++] = page++; 0.00 : ffff800010205950: str x0, [x21, x1] 0.00 : ffff800010205954: add x0, x0, #0x40 : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff800010205958: b.ne ffff800010205938 // b.any 0.00 : ffff80001020595c: ldr x1, [x6, #1880] : gup_huge_pud(): : head = try_get_compound_head(pud_page(orig), refs); 0.00 : ffff800010205960: add x4, x1, x4, lsl #6 : __read_once_size(): 0.00 : ffff800010205964: ldr x0, [x4, #8] : compound_head(): 0.00 : ffff800010205968: sub x1, x0, #0x1 0.00 : ffff80001020596c: tst x0, #0x1 0.00 : ffff800010205970: csel x4, x1, x4, ne // ne = any : __read_once_size(): 0.00 : ffff800010205974: ldr w0, [x4, #52] : try_get_compound_head(): : if (WARN_ON_ONCE(page_ref_count(head) < 0)) 0.00 : ffff800010205978: tbnz w0, #31, ffff800010205734 : __read_once_size(): 0.00 : ffff80001020597c: ldr w6, [x4, #52] : atomic_fetch_add_unless(): 0.00 : ffff800010205980: cbz w6, ffff80001020557c 0.00 : ffff800010205984: add x3, x4, #0x34 : atomic_cmpxchg(): 0.00 : ffff800010205988: sxtw x1, w6 : atomic_fetch_add_unless(): : } while (!atomic_try_cmpxchg(v, &c, c + a)); 0.00 : ffff80001020598c: add w2, w6, w5 : arch_static_branch_jump(): 0.00 : ffff800010205990: b ffff8000102059bc 0.00 : ffff800010205994: b ffff8000102059bc : __lse__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 0.00 : ffff800010205998: mov x0, x3 0.00 : ffff80001020599c: mov w7, w1 0.00 : ffff8000102059a0: casal w7, w2, [x3] 0.00 : ffff8000102059a4: mov w0, w7 : atomic_try_cmpxchg(): : if (unlikely(r != o)) 0.00 : ffff8000102059a8: cmp w0, w6 0.00 : ffff8000102059ac: b.eq ffff8000102059d0 // b.none 0.00 : ffff8000102059b0: mov w6, w0 : atomic_fetch_add_unless(): : if (unlikely(c == u)) 0.00 : ffff8000102059b4: cbnz w0, ffff800010205988 0.00 : ffff8000102059b8: b ffff80001020557c : __ll_sc__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 0.00 : ffff8000102059bc: and x0, x1, #0xffffffff 0.00 : ffff8000102059c0: b ffff800010207c74 0.00 : ffff8000102059c4: mov w0, w1 : atomic_try_cmpxchg(): : if (unlikely(r != o)) 0.00 : ffff8000102059c8: cmp w0, w6 0.00 : ffff8000102059cc: b.ne ffff8000102059b0 // b.any : gup_huge_pud(): : if (!head) 0.00 : ffff8000102059d0: cbz x4, ffff80001020557c : if (unlikely(pud_val(orig) != pud_val(*pudp))) { 0.00 : ffff8000102059d4: ldr x0, [x19] 0.00 : ffff8000102059d8: cmp x23, x0 0.00 : ffff8000102059dc: b.ne ffff800010205a38 // b.any : *nr += refs; 0.00 : ffff8000102059e0: ldr w0, [x26] 0.00 : ffff8000102059e4: add w5, w0, w5 0.00 : ffff8000102059e8: str w5, [x26] : __read_once_size(): 0.00 : ffff8000102059ec: ldr x0, [x4, #8] : compound_head(): 0.00 : ffff8000102059f0: sub x1, x0, #0x1 0.00 : ffff8000102059f4: tst x0, #0x1 0.00 : ffff8000102059f8: csel x4, x1, x4, ne // ne = any : arch_static_branch_jump(): 0.00 : ffff8000102059fc: b ffff800010205a10 0.00 : ffff800010205a00: b ffff800010205a10 : __lse_atomic64_or(): : ATOMIC64_OP(or, stset) 0.00 : ffff800010205a04: mov x0, #0x2 // #2 0.00 : ffff800010205a08: stset x0, [x4] 0.00 : ffff800010205a0c: b ffff8000102056ac : __ll_sc_atomic64_or(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010205a10: b ffff800010207c94 0.00 : ffff800010205a14: b ffff8000102056ac : record_subpages(): : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff800010205a18: mov w6, #0x0 // #0 0.00 : ffff800010205a1c: b ffff800010205820 : gup_huge_pmd(): : put_compound_head(head, refs); 0.00 : ffff800010205a20: mov w1, w6 0.00 : ffff800010205a24: mov x0, x5 0.00 : ffff800010205a28: bl ffff800010205390 0.00 : ffff800010205a2c: b ffff80001020557c : record_subpages(): : for (nr = 0; addr != end; addr += PAGE_SIZE) 0.00 : ffff800010205a30: mov w5, #0x0 // #0 0.00 : ffff800010205a34: b ffff800010205960 : gup_huge_pud(): : put_compound_head(head, refs); 0.00 : ffff800010205a38: mov w1, w5 0.00 : ffff800010205a3c: mov x0, x4 0.00 : ffff800010205a40: bl ffff800010205390 0.00 : ffff800010205a44: b ffff80001020557c Percent | Source code & Disassembly of vmlinux for cycles (184 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd2ed8 : : schedule_idle(): : * : * schedule_idle() is similar to schedule_preempt_disable() except that it : * never enables preemption because it does not call sched_submit_work(). : */ : void __sched schedule_idle(void) : { 3.73 : ffff800010cd2ed8: stp x29, x30, [sp, #-16]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.56 : ffff800010cd2edc: mrs x0, sp_el0 : schedule_idle(): 0.00 : ffff800010cd2ee0: mov x29, sp : * regardless because that function is a nop when the task is in a : * TASK_RUNNING state, make sure this isn't used someplace that the : * current task can be in any other state. Note, idle is always in the : * TASK_RUNNING state. : */ : WARN_ON_ONCE(current->state); 0.00 : ffff800010cd2ee4: ldr x0, [x0, #24] 1.04 : ffff800010cd2ee8: cbnz x0, ffff800010cd2f08 : do { : __schedule(false); 3.19 : ffff800010cd2eec: mov w0, #0x0 // #0 0.00 : ffff800010cd2ef0: bl ffff800010cd24e0 <__schedule> : get_current(): 25.14 : ffff800010cd2ef4: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.54 : ffff800010cd2ef8: ldr x0, [x0] : schedule_idle(): : } while (need_resched()); 0.00 : ffff800010cd2efc: tbnz w0, #1, ffff800010cd2eec : } 65.79 : ffff800010cd2f00: ldp x29, x30, [sp], #16 0.00 : ffff800010cd2f04: ret : WARN_ON_ONCE(current->state); 0.00 : ffff800010cd2f08: brk #0x800 0.00 : ffff800010cd2f0c: b ffff800010cd2eec Percent | Source code & Disassembly of vmlinux for cycles (311 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010084e74 : : cpu_switch_to(): : * x1 = next task_struct : * Previous and next are guaranteed not to be the same. : * : */ : ENTRY(cpu_switch_to) : mov x10, #THREAD_CPU_CONTEXT 0.00 : ffff800010084e74: mov x10, #0x9c0 // #2496 : add x8, x0, x10 0.00 : ffff800010084e78: add x8, x0, x10 : mov x9, sp 0.00 : ffff800010084e7c: mov x9, sp : stp x19, x20, [x8], #16 // store callee-saved registers 0.00 : ffff800010084e80: stp x19, x20, [x8], #16 : stp x21, x22, [x8], #16 63.62 : ffff800010084e84: stp x21, x22, [x8], #16 : stp x23, x24, [x8], #16 3.31 : ffff800010084e88: stp x23, x24, [x8], #16 : stp x25, x26, [x8], #16 2.30 : ffff800010084e8c: stp x25, x26, [x8], #16 : stp x27, x28, [x8], #16 3.55 : ffff800010084e90: stp x27, x28, [x8], #16 : stp x29, x9, [x8], #16 2.62 : ffff800010084e94: stp x29, x9, [x8], #16 : str lr, [x8] 3.50 : ffff800010084e98: str x30, [x8] : add x8, x1, x10 0.00 : ffff800010084e9c: add x8, x1, x10 : ldp x19, x20, [x8], #16 // restore callee-saved registers 2.26 : ffff800010084ea0: ldp x19, x20, [x8], #16 : ldp x21, x22, [x8], #16 6.65 : ffff800010084ea4: ldp x21, x22, [x8], #16 : ldp x23, x24, [x8], #16 6.77 : ffff800010084ea8: ldp x23, x24, [x8], #16 : ldp x25, x26, [x8], #16 2.26 : ffff800010084eac: ldp x25, x26, [x8], #16 : ldp x27, x28, [x8], #16 0.00 : ffff800010084eb0: ldp x27, x28, [x8], #16 : ldp x29, x9, [x8], #16 0.00 : ffff800010084eb4: ldp x29, x9, [x8], #16 : ldr lr, [x8] 2.54 : ffff800010084eb8: ldr x30, [x8] : mov sp, x9 0.00 : ffff800010084ebc: mov sp, x9 : msr sp_el0, x1 0.62 : ffff800010084ec0: msr sp_el0, x1 : ret 0.00 : ffff800010084ec4: ret Percent | Source code & Disassembly of vmlinux for cycles (174 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100ecde0 : : raise_softirq(): : if (!in_interrupt()) : wakeup_softirqd(); : } : : void raise_softirq(unsigned int nr) : { 0.00 : ffff8000100ecde0: stp x29, x30, [sp, #-32]! 0.00 : ffff8000100ecde4: mov x29, sp 0.59 : ffff8000100ecde8: str x19, [sp, #16] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000100ecdec: mrs x19, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000100ecdf0: and w1, w19, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff8000100ecdf4: cbnz w1, ffff8000100ece00 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff8000100ecdf8: mov x1, #0x60 // #96 0.00 : ffff8000100ecdfc: msr daifset, #0x2 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000100ece00: mrs x4, tpidr_el1 : __raise_softirq_irqoff(): : } : : void __raise_softirq_irqoff(unsigned int nr) : { : trace_softirq_raise(nr); : or_softirq_pending(1UL << nr); 0.00 : ffff8000100ece04: adrp x1, ffff800011528000 0.00 : ffff8000100ece08: add x1, x1, #0x7c0 0.00 : ffff8000100ece0c: ldr w2, [x1, x4] 0.00 : ffff8000100ece10: mov x3, #0x1 // #1 0.00 : ffff8000100ece14: lsl x0, x3, x0 0.00 : ffff8000100ece18: orr w0, w2, w0 0.00 : ffff8000100ece1c: str w0, [x1, x4] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000100ece20: mrs x3, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000100ece24: ldr w0, [x3, #16] : raise_softirq_irqoff(): : if (!in_interrupt()) 0.00 : ffff8000100ece28: tst w0, #0x1fff00 0.00 : ffff8000100ece2c: b.ne ffff8000100ece34 // b.any : wakeup_softirqd(); 0.00 : ffff8000100ece30: bl ffff8000100ec5c0 : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff8000100ece34: msr daif, x19 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 9.28 : ffff8000100ece38: nop : raise_softirq(): : } 0.00 : ffff8000100ece3c: ldr x19, [sp, #16] 0.00 : ffff8000100ece40: ldp x29, x30, [sp], #32 0.00 : ffff8000100ece44: ret : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.57 : ffff8000100ece48: dsb sy : raise_softirq(): 78.08 : ffff8000100ece4c: ldr x19, [sp, #16] 11.47 : ffff8000100ece50: ldp x29, x30, [sp], #32 0.00 : ffff8000100ece54: ret Percent | Source code & Disassembly of vmlinux for cycles (178 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010127460 : : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010127460: add w1, w0, #0x3f 0.00 : ffff800010127464: cmp w0, #0x0 0.00 : ffff800010127468: csel w1, w1, w0, lt // lt = tstop : nohz_balance_enter_idle(): : /* : * This routine will record that the CPU is going idle with tick stopped. : * This info will be used in performing idle load balancing in the future. : */ : void nohz_balance_enter_idle(int cpu) : { 0.00 : ffff80001012746c: stp x29, x30, [sp, #-64]! : test_bit(): 0.00 : ffff800010127470: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010127474: add x2, x2, #0x1c8 0.00 : ffff800010127478: asr w1, w1, #6 : nohz_balance_enter_idle(): 0.00 : ffff80001012747c: mov x29, sp : test_bit(): 0.00 : ffff800010127480: sxtw x1, w1 : nohz_balance_enter_idle(): 0.00 : ffff800010127484: str x19, [sp, #16] 0.00 : ffff800010127488: str x21, [sp, #32] : test_bit(): 0.00 : ffff80001012748c: and w21, w0, #0x3f : nohz_balance_enter_idle(): : struct rq *rq = cpu_rq(cpu); 0.00 : ffff800010127490: adrp x19, ffff800011528000 0.00 : ffff800010127494: add x19, x19, #0xe80 : test_bit(): 0.00 : ffff800010127498: ldr x1, [x2, x1, lsl #3] 0.00 : ffff80001012749c: lsr x1, x1, x21 : nohz_balance_enter_idle(): : : SCHED_WARN_ON(cpu != smp_processor_id()); : : /* If this CPU is going down, then nothing needs to be done: */ : if (!cpu_active(cpu)) 0.00 : ffff8000101274a0: tbnz w1, #0, ffff8000101274b4 : /* : * Each time a cpu enter idle, we assume that it has blocked load and : * enable the periodic update of the load of idle cpus : */ : WRITE_ONCE(nohz.has_blocked, 1); : } 0.00 : ffff8000101274a4: ldr x19, [sp, #16] 0.00 : ffff8000101274a8: ldr x21, [sp, #32] 0.00 : ffff8000101274ac: ldp x29, x30, [sp], #64 0.00 : ffff8000101274b0: ret 1.13 : ffff8000101274b4: stp x22, x23, [x29, #40] : struct rq *rq = cpu_rq(cpu); 0.00 : ffff8000101274b8: adrp x23, ffff800011909000 39.52 : ffff8000101274bc: str x24, [x29, #56] 0.00 : ffff8000101274c0: sxtw x24, w0 0.00 : ffff8000101274c4: add x0, x23, #0x928 0.00 : ffff8000101274c8: str x20, [x29, #24] 0.00 : ffff8000101274cc: mov x20, x24 0.00 : ffff8000101274d0: ldr x22, [x0, x24, lsl #3] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101274d4: nop : nohz_balance_enter_idle(): 0.00 : ffff8000101274d8: add x19, x22, x19 : rq->has_blocked_load = 1; 0.00 : ffff8000101274dc: mov w2, #0x1 // #1 : if (rq->nohz_tick_stopped) 0.00 : ffff8000101274e0: ldr w0, [x19, #44] : rq->has_blocked_load = 1; 10.53 : ffff8000101274e4: str w2, [x19, #40] : if (rq->nohz_tick_stopped) 0.00 : ffff8000101274e8: cbz w0, ffff800010127548 0.00 : ffff8000101274ec: adrp x20, ffff800011afb000 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101274f0: add x0, x20, #0xfc0 0.00 : ffff8000101274f4: mov w1, #0x1 // #1 1.70 : ffff8000101274f8: str w1, [x0, #36] 0.56 : ffff8000101274fc: ldr x20, [x29, #24] 0.55 : ffff800010127500: ldp x22, x23, [x29, #40] 0.00 : ffff800010127504: ldr x24, [x29, #56] : nohz_balance_enter_idle(): : } 4.42 : ffff800010127508: ldr x19, [sp, #16] 0.00 : ffff80001012750c: ldr x21, [sp, #32] 0.00 : ffff800010127510: ldp x29, x30, [sp], #64 0.00 : ffff800010127514: ret : housekeeping_cpu(): : : static inline bool housekeeping_cpu(int cpu, enum hk_flags flags) : { : #ifdef CONFIG_CPU_ISOLATION : if (static_branch_unlikely(&housekeeping_overridden)) : return housekeeping_test_cpu(cpu, flags); 0.00 : ffff800010127518: mov w1, #0x8 // #8 0.00 : ffff80001012751c: mov w0, w24 0.00 : ffff800010127520: bl ffff800010137e78 : nohz_balance_enter_idle(): : if (!housekeeping_cpu(cpu, HK_FLAG_SCHED)) 0.00 : ffff800010127524: tst w0, #0xff 0.00 : ffff800010127528: b.ne ffff8000101274d8 // b.any 0.00 : ffff80001012752c: ldr x20, [x29, #24] 0.00 : ffff800010127530: ldp x22, x23, [x29, #40] 0.00 : ffff800010127534: ldr x24, [x29, #56] : } 0.00 : ffff800010127538: ldr x19, [sp, #16] 0.00 : ffff80001012753c: ldr x21, [sp, #32] 0.00 : ffff800010127540: ldp x29, x30, [sp], #64 0.00 : ffff800010127544: ret : __read_once_size(): : __READ_ONCE_SIZE; 1.68 : ffff800010127548: ldr x0, [x19, #2472] : nohz_balance_enter_idle(): : if (on_null_domain(rq)) 0.00 : ffff80001012754c: cbz x0, ffff80001012752c : set_bit(): : * See Documentation/atomic_bitops.txt for details. : */ : : static inline void set_bit(unsigned int nr, volatile unsigned long *p) : { : p += BIT_WORD(nr); 0.57 : ffff800010127550: lsr w0, w20, #6 : atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff800010127554: mov x1, #0x1 // #1 : p += BIT_WORD(nr); 0.00 : ffff800010127558: adrp x20, ffff800011afb000 0.00 : ffff80001012755c: add x3, x20, #0xfc0 0.00 : ffff800010127560: add x0, x3, x0, lsl #3 : atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff800010127564: lsl x21, x1, x21 : nohz_balance_enter_idle(): : rq->nohz_tick_stopped = 1; 0.00 : ffff800010127568: str w2, [x19, #44] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001012756c: b ffff800010127594 0.00 : ffff800010127570: b ffff800010127594 : __lse_atomic64_or(): : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) 0.00 : ffff800010127574: stset x21, [x0] : arch_static_branch_jump(): 1.11 : ffff800010127578: b ffff8000101275a0 0.00 : ffff80001012757c: b ffff8000101275a0 : __lse_atomic_add(): : ATOMIC_OP(add, stadd) 0.00 : ffff800010127580: add x0, x20, #0xfc0 0.00 : ffff800010127584: mov w1, #0x1 // #1 0.00 : ffff800010127588: add x2, x0, #0x20 0.00 : ffff80001012758c: stadd w1, [x2] 0.57 : ffff800010127590: b ffff8000101275ac : __ll_sc_atomic64_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010127594: b ffff800010128ba4 : arch_static_branch_jump(): 0.00 : ffff800010127598: b ffff8000101275a0 0.00 : ffff80001012759c: b ffff80001012757c : __ll_sc_atomic_add(): : ATOMIC_OPS(add, add, I) 0.00 : ffff8000101275a0: add x0, x20, #0xfc0 0.00 : ffff8000101275a4: add x0, x0, #0x20 0.00 : ffff8000101275a8: b ffff800010128bbc : nohz_balance_enter_idle(): : smp_mb__after_atomic(); 0.00 : ffff8000101275ac: dmb ish : set_cpu_sd_state_idle(): : sd = rcu_dereference(per_cpu(sd_llc, cpu)); 37.66 : ffff8000101275b0: add x23, x23, #0x928 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000101275b4: bl ffff80001015c1b0 <__rcu_read_lock> : set_cpu_sd_state_idle(): 0.00 : ffff8000101275b8: adrp x0, ffff80001151f000 0.00 : ffff8000101275bc: add x0, x0, #0x118 0.00 : ffff8000101275c0: ldr x1, [x23, x24, lsl #3] : __read_once_size(): 0.00 : ffff8000101275c4: ldr x0, [x0, x1] : set_cpu_sd_state_idle(): : if (!sd || sd->nohz_idle) 0.00 : ffff8000101275c8: cbz x0, ffff8000101275d4 0.00 : ffff8000101275cc: ldr w1, [x0, #52] 0.00 : ffff8000101275d0: cbz w1, ffff8000101275dc : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff8000101275d4: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff8000101275d8: b ffff8000101274f0 : set_cpu_sd_state_idle(): : atomic_dec(&sd->shared->nr_busy_cpus); 0.00 : ffff8000101275dc: ldr x2, [x0, #120] : sd->nohz_idle = 1; 0.00 : ffff8000101275e0: mov w1, #0x1 // #1 0.00 : ffff8000101275e4: str w1, [x0, #52] : atomic_dec(&sd->shared->nr_busy_cpus); 0.00 : ffff8000101275e8: add x3, x2, #0x4 : arch_atomic_sub(): : ATOMIC_OP(atomic_andnot) : ATOMIC_OP(atomic_or) : ATOMIC_OP(atomic_xor) : ATOMIC_OP(atomic_add) : ATOMIC_OP(atomic_and) : ATOMIC_OP(atomic_sub) 0.00 : ffff8000101275ec: bl ffff80001011cba0 0.00 : ffff8000101275f0: tst w0, #0xff 0.00 : ffff8000101275f4: b.ne ffff800010127608 // b.any : __ll_sc_atomic_sub(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000101275f8: add x2, x2, #0x4 0.00 : ffff8000101275fc: b ffff800010128bd4 : rcu_read_unlock(): 0.00 : ffff800010127600: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff800010127604: b ffff8000101274f0 : __lse_atomic_sub(): : asm volatile( 0.00 : ffff800010127608: add x0, x2, #0x4 0.00 : ffff80001012760c: neg w1, w1 0.00 : ffff800010127610: stadd w1, [x0] : rcu_read_unlock(): 0.00 : ffff800010127614: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff800010127618: b ffff8000101274f0 Percent | Source code & Disassembly of vmlinux for cycles (211 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010136480 : : cpuacct_charge(): : * charge this task's execution time to its accounting group. : * : * called with rq->lock held. : */ : void cpuacct_charge(struct task_struct *tsk, u64 cputime) : { 1.97 : ffff800010136480: stp x29, x30, [sp, #-48]! 0.00 : ffff800010136484: mov x29, sp 15.13 : ffff800010136488: stp x19, x20, [sp, #16] 0.00 : ffff80001013648c: mov x20, x1 4.73 : ffff800010136490: str x21, [sp, #32] 0.00 : ffff800010136494: mov w19, #0x1 // #1 0.00 : ffff800010136498: mov x21, x0 : task_stack_page(): : * try_get_task_stack() instead. task_stack_page will return a pointer : * that could get freed out from under you. : */ : static inline void *task_stack_page(const struct task_struct *task) : { : return task->stack; 6.46 : ffff80001013649c: ldr x1, [x0, #32] : cpuacct_charge(): : struct cpuacct *ca; : int index = CPUACCT_STAT_SYSTEM; : struct pt_regs *regs = task_pt_regs(tsk); 0.00 : ffff8000101364a0: mov x0, #0x3ec0 // #16064 0.00 : ffff8000101364a4: add x1, x1, x0 : : if (regs && user_mode(regs)) 0.00 : ffff8000101364a8: cbz x1, ffff8000101364b8 1.49 : ffff8000101364ac: ldr x0, [x1, #264] 0.00 : ffff8000101364b0: tst x0, #0xf 0.00 : ffff8000101364b4: cset w19, ne // ne = any : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000101364b8: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.95 : ffff8000101364bc: ldr x0, [x21, #2000] : task_css(): : * See task_css_check(). : */ : static inline struct cgroup_subsys_state *task_css(struct task_struct *task, : int subsys_id) : { : return task_css_check(task, subsys_id, false); 2.00 : ffff8000101364c0: ldr x2, [x0, #16] : css_ca(): : return css ? container_of(css, struct cpuacct, css) : NULL; 0.00 : ffff8000101364c4: cbz x2, ffff8000101364f0 15.67 : ffff8000101364c8: sxtw x4, w19 0.00 : ffff8000101364cc: lsl x4, x4, #3 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 1.43 : ffff8000101364d0: mrs x3, tpidr_el1 : cpuacct_charge(): : index = CPUACCT_STAT_USER; : : rcu_read_lock(); : : for (ca = task_ca(tsk); ca; ca = parent_ca(ca)) : this_cpu_ptr(ca->cpuusage)->usages[index] += cputime; 0.00 : ffff8000101364d4: ldr x0, [x2, #240] 0.00 : ffff8000101364d8: add x0, x0, x3 21.65 : ffff8000101364dc: ldr x3, [x0, x4] 0.50 : ffff8000101364e0: add x3, x3, x20 25.73 : ffff8000101364e4: str x3, [x0, x4] : for (ca = task_ca(tsk); ca; ca = parent_ca(ca)) 0.00 : ffff8000101364e8: ldr x2, [x2, #232] : css_ca(): : return css ? container_of(css, struct cpuacct, css) : NULL; 0.00 : ffff8000101364ec: cbnz x2, ffff8000101364d0 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff8000101364f0: bl ffff800010160d20 <__rcu_read_unlock> : cpuacct_charge(): : : rcu_read_unlock(); : } 0.00 : ffff8000101364f4: ldr x21, [sp, #32] 0.93 : ffff8000101364f8: ldp x19, x20, [sp, #16] 1.37 : ffff8000101364fc: ldp x29, x30, [sp], #48 0.00 : ffff800010136500: ret Percent | Source code & Disassembly of vmlinux for cycles (422 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010716be0 <__arm_lpae_map>: : __arm_lpae_map(): : } : : static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, : phys_addr_t paddr, size_t size, arm_lpae_iopte prot, : int lvl, arm_lpae_iopte *ptep) : { 1.20 : ffff800010716be0: stp x29, x30, [sp, #-112]! : arm_lpae_iopte *cptep, pte; : size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 0.00 : ffff800010716be4: mov w7, #0x4 // #4 0.00 : ffff800010716be8: sub w7, w7, w5 : { 0.00 : ffff800010716bec: mov x29, sp 1.44 : ffff800010716bf0: stp x19, x20, [sp, #16] 0.00 : ffff800010716bf4: mov x20, x0 0.72 : ffff800010716bf8: stp x21, x22, [sp, #32] 0.00 : ffff800010716bfc: mov w21, w5 0.00 : ffff800010716c00: stp x23, x24, [sp, #48] : size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 0.00 : ffff800010716c04: mov x22, #0x1 // #1 : { 0.48 : ffff800010716c08: stp x25, x26, [sp, #64] 0.00 : ffff800010716c0c: mov x25, x2 0.68 : ffff800010716c10: str x27, [sp, #80] 0.00 : ffff800010716c14: mov x23, x1 0.00 : ffff800010716c18: mov x24, x3 0.00 : ffff800010716c1c: mov x26, x4 : size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 1.45 : ffff800010716c20: ldr w0, [x0, #120] : size_t tblsz = ARM_LPAE_GRANULE(data); : struct io_pgtable_cfg *cfg = &data->iop.cfg; : : /* Find our entry at the current level */ : ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 1.44 : ffff800010716c24: ldr w5, [x20, #116] 0.00 : ffff800010716c28: cmp w5, w21 : size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 0.00 : ffff800010716c2c: mul w2, w7, w0 0.00 : ffff800010716c30: add w2, w2, #0x3 : ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 0.73 : ffff800010716c34: lsr x7, x1, x2 0.00 : ffff800010716c38: mov w1, w0 : size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 0.00 : ffff800010716c3c: lsl x22, x22, x2 : ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 0.00 : ffff800010716c40: b.ne ffff800010716c48 <__arm_lpae_map+0x68> // b.any 0.00 : ffff800010716c44: ldr w1, [x20, #112] 0.24 : ffff800010716c48: mov w19, #0x1 // #1 : : /* If we can install a leaf entry at this level, then do so */ : if (size == block_size) 0.00 : ffff800010716c4c: cmp x22, x24 : ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); 0.00 : ffff800010716c50: lsl w19, w19, w1 1.44 : ffff800010716c54: sub w19, w19, #0x1 1.69 : ffff800010716c58: sxtw x19, w19 0.00 : ffff800010716c5c: and x19, x19, x7 0.00 : ffff800010716c60: add x27, x6, x19, lsl #3 : if (size == block_size) 0.00 : ffff800010716c64: b.eq ffff800010716d6c <__arm_lpae_map+0x18c> // b.none : return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); : : /* We can't allocate tables at the final level */ : if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) 0.24 : ffff800010716c68: cmp w21, #0x2 0.00 : ffff800010716c6c: b.gt ffff800010716e80 <__arm_lpae_map+0x2a0> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 3.34 : ffff800010716c70: ldr x19, [x6, x19, lsl #3] : __arm_lpae_map(): : return -EINVAL; : : /* Grab a pointer to the next level */ : pte = READ_ONCE(*ptep); : if (!pte) { 0.00 : ffff800010716c74: cbz x19, ffff800010716cb8 <__arm_lpae_map+0xd8> : return -ENOMEM; : : pte = arm_lpae_install_table(cptep, ptep, 0, cfg); : if (pte) : __arm_lpae_free_pages(cptep, tblsz, cfg); : } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { 21.82 : ffff800010716c78: ldrb w0, [x20, #40] 0.00 : ffff800010716c7c: cbnz w0, ffff800010716c84 <__arm_lpae_map+0xa4> 0.00 : ffff800010716c80: tbz x19, #55, ffff800010716d40 <__arm_lpae_map+0x160> : iopte_leaf(): : return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK; 1.92 : ffff800010716c84: and x0, x19, #0x3 : __arm_lpae_map(): : __arm_lpae_sync_pte(ptep, cfg); : } : : if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) { 0.00 : ffff800010716c88: cmp x0, #0x1 0.00 : ffff800010716c8c: b.ne ffff800010716da4 <__arm_lpae_map+0x1c4> // b.any : cptep = iopte_deref(pte, data); : } else if (pte) { : /* We require an unmap first */ : WARN_ON(!selftest_running); 0.00 : ffff800010716c90: brk #0x800 : return -EEXIST; 0.00 : ffff800010716c94: mov w19, #0xffffffef // #-17 : } : : /* Rinse, repeat */ : return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); : } 0.47 : ffff800010716c98: mov w0, w19 0.00 : ffff800010716c9c: ldr x27, [sp, #80] 0.24 : ffff800010716ca0: ldp x19, x20, [sp, #16] 0.24 : ffff800010716ca4: ldp x21, x22, [sp, #32] 0.24 : ffff800010716ca8: ldp x23, x24, [sp, #48] 0.00 : ffff800010716cac: ldp x25, x26, [sp, #64] 0.00 : ffff800010716cb0: ldp x29, x30, [sp], #112 0.00 : ffff800010716cb4: ret 0.00 : ffff800010716cb8: str x28, [x29, #88] : size_t tblsz = ARM_LPAE_GRANULE(data); 0.00 : ffff800010716cbc: mov x28, #0x8 // #8 : cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg); 0.00 : ffff800010716cc0: add x2, x20, #0x28 0.00 : ffff800010716cc4: mov w1, #0xa20 // #2592 0.00 : ffff800010716cc8: ldr x3, [x20, #56] : size_t tblsz = ARM_LPAE_GRANULE(data); 0.00 : ffff800010716ccc: lsl x28, x28, x0 : cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg); 0.00 : ffff800010716cd0: mov x0, x28 0.00 : ffff800010716cd4: bl ffff800010715be8 <__arm_lpae_alloc_pages.isra.22> 0.00 : ffff800010716cd8: mov x22, x0 : if (!cptep) 0.00 : ffff800010716cdc: cbz x0, ffff800010716e8c <__arm_lpae_map+0x2ac> : pte = arm_lpae_install_table(cptep, ptep, 0, cfg); 0.00 : ffff800010716ce0: add x3, x20, #0x10 0.00 : ffff800010716ce4: mov x2, #0x0 // #0 0.00 : ffff800010716ce8: mov x1, x27 0.00 : ffff800010716cec: bl ffff800010715b10 0.00 : ffff800010716cf0: mov x19, x0 : if (pte) 0.00 : ffff800010716cf4: cbnz x0, ffff800010716d50 <__arm_lpae_map+0x170> 0.00 : ffff800010716cf8: ldr x28, [x29, #88] : return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); 0.92 : ffff800010716cfc: mov x6, x22 0.24 : ffff800010716d00: add w5, w21, #0x1 0.00 : ffff800010716d04: mov x4, x26 0.00 : ffff800010716d08: mov x3, x24 0.00 : ffff800010716d0c: mov x2, x25 0.00 : ffff800010716d10: mov x1, x23 0.00 : ffff800010716d14: mov x0, x20 0.00 : ffff800010716d18: bl ffff800010716be0 <__arm_lpae_map> 0.00 : ffff800010716d1c: mov w19, w0 : } 0.48 : ffff800010716d20: ldr x27, [sp, #80] 0.00 : ffff800010716d24: mov w0, w19 0.96 : ffff800010716d28: ldp x19, x20, [sp, #16] 0.24 : ffff800010716d2c: ldp x21, x22, [sp, #32] 0.69 : ffff800010716d30: ldp x23, x24, [sp, #48] 0.47 : ffff800010716d34: ldp x25, x26, [sp, #64] 0.24 : ffff800010716d38: ldp x29, x30, [sp], #112 0.00 : ffff800010716d3c: ret : __arm_lpae_sync_pte(ptep, cfg); 0.00 : ffff800010716d40: ldr x1, [x20, #56] 0.00 : ffff800010716d44: mov x0, x27 0.00 : ffff800010716d48: bl ffff800010715a88 <__arm_lpae_sync_pte.isra.21> 0.00 : ffff800010716d4c: b ffff800010716c84 <__arm_lpae_map+0xa4> : __arm_lpae_free_pages(cptep, tblsz, cfg); 0.00 : ffff800010716d50: ldrb w2, [x20, #40] 0.00 : ffff800010716d54: mov x1, x28 0.00 : ffff800010716d58: add x3, x20, #0x38 0.00 : ffff800010716d5c: mov x0, x22 0.00 : ffff800010716d60: bl ffff800010716518 <__arm_lpae_free_pages.isra.20> 0.00 : ffff800010716d64: ldr x28, [x29, #88] 0.00 : ffff800010716d68: b ffff800010716c84 <__arm_lpae_map+0xa4> : iopte_leaf(): : if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE) 0.00 : ffff800010716d6c: ldr w2, [x20] 0.00 : ffff800010716d70: cmp w21, #0x3 0.95 : ffff800010716d74: ldr x1, [x6, x19, lsl #3] 0.00 : ffff800010716d78: cset w24, eq // eq = none 0.00 : ffff800010716d7c: cmp w2, #0x5 0.00 : ffff800010716d80: and x1, x1, #0x3 49.06 : ffff800010716d84: csel w2, w24, wzr, ne // ne = any 0.00 : ffff800010716d88: cbz w2, ffff800010716dd8 <__arm_lpae_map+0x1f8> : return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_PAGE; 0.00 : ffff800010716d8c: cmp x1, #0x3 0.00 : ffff800010716d90: cset w3, eq // eq = none : arm_lpae_init_pte(): : if (iopte_leaf(pte, lvl, data->iop.fmt)) { 0.00 : ffff800010716d94: cbz w3, ffff800010716de4 <__arm_lpae_map+0x204> : WARN_ON(!selftest_running); 0.00 : ffff800010716d98: brk #0x800 : return -EEXIST; 0.00 : ffff800010716d9c: mov w19, #0xffffffef // #-17 0.00 : ffff800010716da0: b ffff800010716c98 <__arm_lpae_map+0xb8> : iopte_to_paddr(): : u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716da4: and x19, x19, #0xfffffffff000 : if (ARM_LPAE_GRANULE(data) < SZ_64K) 2.40 : ffff800010716da8: ldr w4, [x20, #120] : __arm_lpae_map(): : cptep = iopte_deref(pte, data); 0.00 : ffff800010716dac: adrp x2, ffff8000112f2000 : iopte_to_paddr(): : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716db0: mov x1, #0x8 // #8 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716db4: orr x0, x19, x19, lsl #36 : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716db8: mov x3, #0xffff // #65535 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716dbc: and x0, x0, #0xfffffffff0000 : __arm_lpae_map(): : cptep = iopte_deref(pte, data); 0.00 : ffff800010716dc0: ldr x22, [x2, #1872] : iopte_to_paddr(): : if (ARM_LPAE_GRANULE(data) < SZ_64K) 0.00 : ffff800010716dc4: lsl x1, x1, x4 : return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 0.00 : ffff800010716dc8: cmp x1, x3 0.00 : ffff800010716dcc: csel x19, x0, x19, hi // hi = pmore : __arm_lpae_map(): : cptep = iopte_deref(pte, data); 0.48 : ffff800010716dd0: sub x22, x19, x22 0.00 : ffff800010716dd4: b ffff800010716cfc <__arm_lpae_map+0x11c> : iopte_leaf(): : return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK; 0.00 : ffff800010716dd8: cmp x1, #0x1 0.00 : ffff800010716ddc: cset w3, eq // eq = none 0.00 : ffff800010716de0: b ffff800010716d94 <__arm_lpae_map+0x1b4> : arm_lpae_init_pte(): : } else if (iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_TABLE) { 0.00 : ffff800010716de4: cmp x1, #0x3 0.00 : ffff800010716de8: b.ne ffff800010716e44 <__arm_lpae_map+0x264> // b.any : tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); 0.00 : ffff800010716dec: cmp w5, w21 0.00 : ffff800010716df0: b.ne ffff800010716df8 <__arm_lpae_map+0x218> // b.any 0.00 : ffff800010716df4: ldr w0, [x20, #112] 0.00 : ffff800010716df8: mov w5, #0x1 // #1 : if (__arm_lpae_unmap(data, NULL, iova, sz, lvl, tblp) != sz) { 0.00 : ffff800010716dfc: mov w4, w21 : tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); 0.00 : ffff800010716e00: lsl w5, w5, w0 0.00 : ffff800010716e04: sub w5, w5, #0x1 : if (__arm_lpae_unmap(data, NULL, iova, sz, lvl, tblp) != sz) { 0.00 : ffff800010716e08: mov x3, x22 0.00 : ffff800010716e0c: mov x2, x23 : tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); 0.00 : ffff800010716e10: sxtw x5, w5 : if (__arm_lpae_unmap(data, NULL, iova, sz, lvl, tblp) != sz) { 0.00 : ffff800010716e14: mov x1, #0x0 // #0 : tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); 0.00 : ffff800010716e18: and x5, x5, x7 : if (__arm_lpae_unmap(data, NULL, iova, sz, lvl, tblp) != sz) { 0.00 : ffff800010716e1c: mov x0, x20 0.00 : ffff800010716e20: str x6, [x29, #104] 0.00 : ffff800010716e24: sub x5, x27, x5, lsl #3 0.00 : ffff800010716e28: bl ffff800010716718 <__arm_lpae_unmap> 0.00 : ffff800010716e2c: cmp x22, x0 0.00 : ffff800010716e30: b.ne ffff800010716e98 <__arm_lpae_map+0x2b8> // b.any 0.00 : ffff800010716e34: ldr w0, [x20] 0.00 : ffff800010716e38: ldr x6, [x29, #104] 0.00 : ffff800010716e3c: cmp w0, #0x5 0.00 : ffff800010716e40: csel w2, w24, wzr, ne // ne = any : __arm_lpae_init_pte(): : pte |= ARM_LPAE_PTE_TYPE_PAGE; 2.37 : ffff800010716e44: orr x0, x26, #0x3 0.00 : ffff800010716e48: cmp w2, #0x0 0.00 : ffff800010716e4c: orr x26, x26, #0x1 : paddr_to_iopte(): : return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716e50: orr x25, x25, x25, lsr #36 : __arm_lpae_init_pte(): : pte |= ARM_LPAE_PTE_TYPE_PAGE; 0.00 : ffff800010716e54: csel x26, x26, x0, eq // eq = none : paddr_to_iopte(): : return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; 0.00 : ffff800010716e58: and x0, x25, #0xfffffffff000 : __arm_lpae_init_pte(): : pte |= paddr_to_iopte(paddr, data); 0.00 : ffff800010716e5c: orr x0, x0, x26 : __arm_lpae_set_pte(): : *ptep = pte; 0.00 : ffff800010716e60: str x0, [x6, x19, lsl #3] : arm_lpae_init_pte(): : return 0; 0.00 : ffff800010716e64: mov w19, #0x0 // #0 : __arm_lpae_set_pte(): : if (!cfg->coherent_walk) 0.48 : ffff800010716e68: ldrb w0, [x20, #40] 0.00 : ffff800010716e6c: cbnz w0, ffff800010716c98 <__arm_lpae_map+0xb8> : __arm_lpae_sync_pte(ptep, cfg); 0.00 : ffff800010716e70: ldr x1, [x20, #56] 0.00 : ffff800010716e74: mov x0, x27 0.00 : ffff800010716e78: bl ffff800010715a88 <__arm_lpae_sync_pte.isra.21> 0.00 : ffff800010716e7c: b ffff800010716c98 <__arm_lpae_map+0xb8> : __arm_lpae_map(): : if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) 0.00 : ffff800010716e80: brk #0x800 : return -EINVAL; 0.00 : ffff800010716e84: mov w19, #0xffffffea // #-22 0.00 : ffff800010716e88: b ffff800010716c98 <__arm_lpae_map+0xb8> : return -ENOMEM; 0.00 : ffff800010716e8c: mov w19, #0xfffffff4 // #-12 0.00 : ffff800010716e90: ldr x28, [x29, #88] 0.00 : ffff800010716e94: b ffff800010716c98 <__arm_lpae_map+0xb8> : arm_lpae_init_pte(): : WARN_ON(1); 0.00 : ffff800010716e98: brk #0x800 : return -EINVAL; 0.00 : ffff800010716e9c: mov w19, #0xffffffea // #-22 0.00 : ffff800010716ea0: b ffff800010716c98 <__arm_lpae_map+0xb8> Percent | Source code & Disassembly of vmlinux for cycles (228 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc4750 : : __rb_erase_augmented(): : static __always_inline struct rb_node * : __rb_erase_augmented(struct rb_node *node, struct rb_root *root, : const struct rb_augment_callbacks *augment) : { : struct rb_node *child = node->rb_right; : struct rb_node *tmp = node->rb_left; 2.25 : ffff800010cc4750: ldp x5, x4, [x0, #8] : struct rb_node *parent, *rebalance; : unsigned long pc; : : if (!tmp) { 0.00 : ffff800010cc4754: cbz x4, ffff800010cc480c : child->__rb_parent_color = pc; : rebalance = NULL; : } else : rebalance = __rb_is_black(pc) ? parent : NULL; : tmp = parent; : } else if (!child) { 0.00 : ffff800010cc4758: cbz x5, ffff800010cc47e8 : rebalance = NULL; : tmp = parent; : } else { : struct rb_node *successor = child, *child2; : : tmp = child->rb_left; 0.00 : ffff800010cc475c: ldr x3, [x5, #16] 0.00 : ffff800010cc4760: mov x6, x5 : if (!tmp) { 0.00 : ffff800010cc4764: mov x2, x5 0.00 : ffff800010cc4768: cbnz x3, ffff800010cc4778 0.00 : ffff800010cc476c: b ffff800010cc499c : */ : do { : parent = successor; : successor = tmp; : tmp = tmp->rb_left; : } while (tmp); 0.00 : ffff800010cc4770: mov x2, x3 0.00 : ffff800010cc4774: mov x3, x4 : tmp = tmp->rb_left; 0.00 : ffff800010cc4778: ldr x4, [x3, #16] : } while (tmp); 0.00 : ffff800010cc477c: cbnz x4, ffff800010cc4770 : child2 = successor->rb_right; 0.00 : ffff800010cc4780: ldr x7, [x3, #8] : rb_set_parent(): : rb->__rb_parent_color = rb_color(rb) | (unsigned long)p; 0.00 : ffff800010cc4784: mov x6, x3 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010cc4788: str x7, [x2, #16] 0.00 : ffff800010cc478c: str x5, [x3, #8] : rb_set_parent(): 0.00 : ffff800010cc4790: ldr x4, [x5] 0.00 : ffff800010cc4794: and x4, x4, #0x1 0.00 : ffff800010cc4798: orr x4, x4, x3 0.00 : ffff800010cc479c: str x4, [x5] 0.00 : ffff800010cc47a0: ldr x4, [x0, #16] : __write_once_size(): 0.00 : ffff800010cc47a4: str x4, [x3, #16] : rb_set_parent(): 0.00 : ffff800010cc47a8: ldr x5, [x4] 0.00 : ffff800010cc47ac: and x5, x5, #0x1 0.00 : ffff800010cc47b0: orr x5, x5, x6 0.00 : ffff800010cc47b4: str x5, [x4] : __rb_erase_augmented(): : : tmp = node->rb_left; : WRITE_ONCE(successor->rb_left, tmp); : rb_set_parent(tmp, successor); : : pc = node->__rb_parent_color; 0.00 : ffff800010cc47b8: ldr x4, [x0] : __rb_change_child(): : if (parent) { 0.00 : ffff800010cc47bc: ands x5, x4, #0xfffffffffffffffc 0.00 : ffff800010cc47c0: b.eq ffff800010cc4834 // b.none : if (parent->rb_left == old) 0.00 : ffff800010cc47c4: ldr x8, [x5, #16] 0.00 : ffff800010cc47c8: cmp x0, x8 0.00 : ffff800010cc47cc: b.eq ffff800010cc4994 // b.none : __write_once_size(): 0.00 : ffff800010cc47d0: str x6, [x5, #8] : __rb_erase_augmented(): : tmp = __rb_parent(pc); : __rb_change_child(node, successor, tmp, root); : : if (child2) { 0.00 : ffff800010cc47d4: cbz x7, ffff800010cc483c : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc47d8: orr x2, x2, #0x1 0.00 : ffff800010cc47dc: str x2, [x7] : __rb_erase_augmented(): : rb_set_parent_color(child2, parent, RB_BLACK); : rebalance = NULL; : } else { : rebalance = rb_is_black(successor) ? parent : NULL; : } : successor->__rb_parent_color = pc; 0.00 : ffff800010cc47e0: str x4, [x3] : rb_erase(): : { : struct rb_node *rebalance; : rebalance = __rb_erase_augmented(node, root, &dummy_callbacks); : if (rebalance) : ____rb_erase_color(rebalance, root, dummy_rotate); : } 0.48 : ffff800010cc47e4: ret : __rb_erase_augmented(): : tmp->__rb_parent_color = pc = node->__rb_parent_color; 0.00 : ffff800010cc47e8: ldr x2, [x0] 0.00 : ffff800010cc47ec: str x2, [x4] : __rb_change_child(): : if (parent) { 0.00 : ffff800010cc47f0: ands x2, x2, #0xfffffffffffffffc 0.00 : ffff800010cc47f4: b.eq ffff800010cc498c // b.none : if (parent->rb_left == old) 0.00 : ffff800010cc47f8: ldr x1, [x2, #16] 0.00 : ffff800010cc47fc: cmp x0, x1 0.00 : ffff800010cc4800: b.eq ffff800010cc4a8c // b.none : __write_once_size(): 0.00 : ffff800010cc4804: str x4, [x2, #8] : rb_erase(): 0.00 : ffff800010cc4808: ret : __rb_erase_augmented(): : pc = node->__rb_parent_color; 46.11 : ffff800010cc480c: ldr x3, [x0] : __rb_change_child(): : if (parent) { 0.00 : ffff800010cc4810: ands x2, x3, #0xfffffffffffffffc 0.00 : ffff800010cc4814: b.eq ffff800010cc497c // b.none : if (parent->rb_left == old) 4.73 : ffff800010cc4818: ldr x4, [x2, #16] 0.00 : ffff800010cc481c: cmp x0, x4 0.94 : ffff800010cc4820: b.eq ffff800010cc4a80 // b.none : __write_once_size(): 0.00 : ffff800010cc4824: str x5, [x2, #8] : __rb_erase_augmented(): : if (child) { 0.00 : ffff800010cc4828: cbz x5, ffff800010cc49d8 : child->__rb_parent_color = pc; 0.00 : ffff800010cc482c: str x3, [x5] : rb_erase(): 0.00 : ffff800010cc4830: ret : __write_once_size(): 0.00 : ffff800010cc4834: str x6, [x1] : __rb_erase_augmented(): : if (child2) { 0.00 : ffff800010cc4838: cbnz x7, ffff800010cc47d8 : rebalance = rb_is_black(successor) ? parent : NULL; 0.00 : ffff800010cc483c: ldr x0, [x3] : successor->__rb_parent_color = pc; 0.00 : ffff800010cc4840: str x4, [x3] : rebalance = rb_is_black(successor) ? parent : NULL; 0.00 : ffff800010cc4844: tbz w0, #0, ffff800010cc47e4 : ____rb_erase_color(): : sibling = parent->rb_right; 0.00 : ffff800010cc4848: ldr x0, [x2, #8] : __rb_erase_augmented(): : child2 = successor->rb_right; 0.00 : ffff800010cc484c: mov x3, #0x0 // #0 : ____rb_erase_color(): : if (node != sibling) { /* node == parent->rb_left */ 0.00 : ffff800010cc4850: cmp x0, x3 0.00 : ffff800010cc4854: b.eq ffff800010cc48e4 // b.none : if (rb_is_red(sibling)) { 0.00 : ffff800010cc4858: ldr x4, [x0] 0.00 : ffff800010cc485c: ldr x3, [x0, #16] 0.00 : ffff800010cc4860: tbnz w4, #0, ffff800010cc48a0 : __write_once_size(): 0.00 : ffff800010cc4864: str x3, [x2, #8] : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4868: orr x4, x2, #0x1 : __write_once_size(): 0.00 : ffff800010cc486c: str x2, [x0, #16] : rb_set_parent_color(): 0.00 : ffff800010cc4870: str x4, [x3] : __rb_rotate_set_parents(): : struct rb_node *parent = rb_parent(old); 0.00 : ffff800010cc4874: ldr x4, [x2] : new->__rb_parent_color = old->__rb_parent_color; 0.00 : ffff800010cc4878: str x4, [x0] : rb_set_parent_color(): 0.00 : ffff800010cc487c: str x0, [x2] : __rb_change_child(): : if (parent) { 0.00 : ffff800010cc4880: ands x4, x4, #0xfffffffffffffffc 0.00 : ffff800010cc4884: b.eq ffff800010cc49a8 // b.none : if (parent->rb_left == old) 0.00 : ffff800010cc4888: ldr x5, [x4, #16] 0.00 : ffff800010cc488c: cmp x2, x5 0.00 : ffff800010cc4890: b.eq ffff800010cc49c8 // b.none : __write_once_size(): 0.00 : ffff800010cc4894: str x0, [x4, #8] : ____rb_erase_color(): : sibling = tmp1; 0.00 : ffff800010cc4898: mov x0, x3 : __write_once_size(): 0.00 : ffff800010cc489c: ldr x3, [x3, #16] : ____rb_erase_color(): : tmp1 = sibling->rb_right; 0.00 : ffff800010cc48a0: ldr x4, [x0, #8] : if (!tmp1 || rb_is_black(tmp1)) { 0.00 : ffff800010cc48a4: cbz x4, ffff800010cc48b0 0.00 : ffff800010cc48a8: ldr x5, [x4] 0.00 : ffff800010cc48ac: tbz w5, #0, ffff800010cc49e0 : if (!tmp2 || rb_is_black(tmp2)) { 0.00 : ffff800010cc48b0: cbz x3, ffff800010cc48bc 0.00 : ffff800010cc48b4: ldr x4, [x3] 0.00 : ffff800010cc48b8: tbz w4, #0, ffff800010cc4ab0 : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc48bc: str x2, [x0] : ____rb_erase_color(): : if (rb_is_red(parent)) 0.00 : ffff800010cc48c0: ldr x0, [x2] 0.00 : ffff800010cc48c4: tbz w0, #0, ffff800010cc4aa4 : if (parent) 0.00 : ffff800010cc48c8: ands x0, x0, #0xfffffffffffffffc 0.00 : ffff800010cc48cc: b.eq ffff800010cc47e4 // b.none 0.00 : ffff800010cc48d0: mov x3, x2 : __rb_erase_augmented(): : child2 = successor->rb_right; 0.00 : ffff800010cc48d4: mov x2, x0 : ____rb_erase_color(): : sibling = parent->rb_right; 0.00 : ffff800010cc48d8: ldr x0, [x2, #8] : if (node != sibling) { /* node == parent->rb_left */ 0.00 : ffff800010cc48dc: cmp x0, x3 0.00 : ffff800010cc48e0: b.ne ffff800010cc4858 // b.any : sibling = parent->rb_left; 0.00 : ffff800010cc48e4: ldr x0, [x2, #16] 0.00 : ffff800010cc48e8: ldp x4, x3, [x0] : if (rb_is_red(sibling)) { 0.00 : ffff800010cc48ec: tbnz w4, #0, ffff800010cc4930 : __write_once_size(): 0.00 : ffff800010cc48f0: str x3, [x2, #16] : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc48f4: orr x4, x2, #0x1 : __write_once_size(): 0.00 : ffff800010cc48f8: str x2, [x0, #8] : rb_set_parent_color(): 0.00 : ffff800010cc48fc: str x4, [x3] : __rb_rotate_set_parents(): : struct rb_node *parent = rb_parent(old); 0.00 : ffff800010cc4900: ldr x4, [x2] : new->__rb_parent_color = old->__rb_parent_color; 0.00 : ffff800010cc4904: str x4, [x0] : rb_set_parent_color(): 0.00 : ffff800010cc4908: str x0, [x2] : __rb_change_child(): : if (parent) { 0.00 : ffff800010cc490c: ands x4, x4, #0xfffffffffffffffc 0.00 : ffff800010cc4910: b.eq ffff800010cc49b8 // b.none : if (parent->rb_left == old) 0.00 : ffff800010cc4914: ldr x5, [x4, #16] 0.00 : ffff800010cc4918: cmp x2, x5 0.00 : ffff800010cc491c: b.eq ffff800010cc4a94 // b.none : __write_once_size(): 0.00 : ffff800010cc4920: str x0, [x4, #8] : ____rb_erase_color(): : sibling = tmp1; 0.00 : ffff800010cc4924: mov x0, x3 : __write_once_size(): 0.00 : ffff800010cc4928: ldr x3, [x3, #8] 0.00 : ffff800010cc492c: nop : ____rb_erase_color(): : tmp1 = sibling->rb_left; 0.00 : ffff800010cc4930: ldr x4, [x0, #16] : if (!tmp1 || rb_is_black(tmp1)) { 0.00 : ffff800010cc4934: cbz x4, ffff800010cc4940 0.00 : ffff800010cc4938: ldr x5, [x4] 0.00 : ffff800010cc493c: tbz w5, #0, ffff800010cc4a30 : if (!tmp2 || rb_is_black(tmp2)) { 0.00 : ffff800010cc4940: cbz x3, ffff800010cc48bc 0.00 : ffff800010cc4944: ldr x4, [x3] 0.00 : ffff800010cc4948: tbnz w4, #0, ffff800010cc48bc : tmp1 = tmp2->rb_left; 0.00 : ffff800010cc494c: ldr x4, [x3, #16] : WRITE_ONCE(parent->rb_left, tmp2); 0.00 : ffff800010cc4950: mov x5, x3 : __write_once_size(): 0.00 : ffff800010cc4954: str x4, [x0, #8] 0.00 : ffff800010cc4958: str x0, [x3, #16] 0.00 : ffff800010cc495c: str x3, [x2, #16] : ____rb_erase_color(): : if (tmp1) 0.00 : ffff800010cc4960: cbz x4, ffff800010cc496c : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4964: orr x6, x0, #0x1 0.00 : ffff800010cc4968: str x6, [x4] 0.00 : ffff800010cc496c: mov x4, x0 0.00 : ffff800010cc4970: mov x0, x3 0.00 : ffff800010cc4974: ldr x3, [x3, #8] 0.00 : ffff800010cc4978: b ffff800010cc4a34 : __write_once_size(): 4.22 : ffff800010cc497c: str x5, [x1] : __rb_erase_augmented(): : if (child) { 1.89 : ffff800010cc4980: cbz x5, ffff800010cc47e4 : child->__rb_parent_color = pc; 0.00 : ffff800010cc4984: str x3, [x5] 0.00 : ffff800010cc4988: b ffff800010cc4830 : __write_once_size(): 0.00 : ffff800010cc498c: str x4, [x1] : rb_erase(): : } 0.00 : ffff800010cc4990: ret : __write_once_size(): 0.00 : ffff800010cc4994: str x6, [x5, #16] 0.00 : ffff800010cc4998: b ffff800010cc47d4 : __rb_erase_augmented(): : child2 = successor->rb_right; 0.00 : ffff800010cc499c: mov x3, x5 0.00 : ffff800010cc49a0: ldr x7, [x5, #8] 0.00 : ffff800010cc49a4: b ffff800010cc47a4 : __write_once_size(): 0.00 : ffff800010cc49a8: str x0, [x1] : ____rb_erase_color(): : sibling = tmp1; 0.00 : ffff800010cc49ac: mov x0, x3 : __write_once_size(): 0.00 : ffff800010cc49b0: ldr x3, [x3, #16] 0.00 : ffff800010cc49b4: b ffff800010cc48a0 0.00 : ffff800010cc49b8: str x0, [x1] : ____rb_erase_color(): : sibling = tmp1; 0.00 : ffff800010cc49bc: mov x0, x3 : __write_once_size(): 0.00 : ffff800010cc49c0: ldr x3, [x3, #8] 0.00 : ffff800010cc49c4: b ffff800010cc4930 0.00 : ffff800010cc49c8: str x0, [x4, #16] : ____rb_erase_color(): : sibling = tmp1; 0.00 : ffff800010cc49cc: mov x0, x3 : __write_once_size(): 0.00 : ffff800010cc49d0: ldr x3, [x3, #16] 0.00 : ffff800010cc49d4: b ffff800010cc48a0 : rb_erase(): : if (rebalance) 2.54 : ffff800010cc49d8: tbnz w3, #0, ffff800010cc4848 : } 0.00 : ffff800010cc49dc: ret 0.00 : ffff800010cc49e0: mov x6, x0 : __write_once_size(): 0.00 : ffff800010cc49e4: str x3, [x2, #8] : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc49e8: orr x5, x6, #0x1 : __write_once_size(): 0.00 : ffff800010cc49ec: str x2, [x0, #16] : rb_set_parent_color(): 0.00 : ffff800010cc49f0: str x5, [x4] : ____rb_erase_color(): : if (tmp2) 0.00 : ffff800010cc49f4: cbz x3, ffff800010cc4a08 : rb_set_parent(): : rb->__rb_parent_color = rb_color(rb) | (unsigned long)p; 0.00 : ffff800010cc49f8: ldr x4, [x3] 0.00 : ffff800010cc49fc: and x4, x4, #0x1 0.00 : ffff800010cc4a00: orr x4, x2, x4 0.00 : ffff800010cc4a04: str x4, [x3] : __rb_rotate_set_parents(): : struct rb_node *parent = rb_parent(old); 0.00 : ffff800010cc4a08: ldr x3, [x2] : new->__rb_parent_color = old->__rb_parent_color; 0.00 : ffff800010cc4a0c: str x3, [x0] : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4a10: str x5, [x2] : __rb_change_child(): : if (parent) { 0.00 : ffff800010cc4a14: ands x3, x3, #0xfffffffffffffffc 0.00 : ffff800010cc4a18: b.eq ffff800010cc4ae0 // b.none : if (parent->rb_left == old) 0.00 : ffff800010cc4a1c: ldr x0, [x3, #16] 0.00 : ffff800010cc4a20: cmp x2, x0 0.00 : ffff800010cc4a24: b.eq ffff800010cc4af0 // b.none : __write_once_size(): 0.00 : ffff800010cc4a28: str x6, [x3, #8] : rb_erase(): : } 0.00 : ffff800010cc4a2c: ret 0.00 : ffff800010cc4a30: mov x5, x0 : __write_once_size(): 0.00 : ffff800010cc4a34: str x3, [x2, #16] : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4a38: orr x6, x5, #0x1 : __write_once_size(): 0.00 : ffff800010cc4a3c: str x2, [x0, #8] : rb_set_parent_color(): 0.00 : ffff800010cc4a40: str x6, [x4] : ____rb_erase_color(): : if (tmp2) 0.00 : ffff800010cc4a44: cbz x3, ffff800010cc4a58 : rb_set_parent(): : rb->__rb_parent_color = rb_color(rb) | (unsigned long)p; 0.00 : ffff800010cc4a48: ldr x4, [x3] 0.00 : ffff800010cc4a4c: and x4, x4, #0x1 0.00 : ffff800010cc4a50: orr x4, x2, x4 0.00 : ffff800010cc4a54: str x4, [x3] : __rb_rotate_set_parents(): : struct rb_node *parent = rb_parent(old); 0.00 : ffff800010cc4a58: ldr x3, [x2] : new->__rb_parent_color = old->__rb_parent_color; 0.00 : ffff800010cc4a5c: str x3, [x0] : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4a60: str x6, [x2] : __rb_change_child(): : if (parent) { 0.00 : ffff800010cc4a64: ands x3, x3, #0xfffffffffffffffc 0.00 : ffff800010cc4a68: b.eq ffff800010cc4ae8 // b.none : if (parent->rb_left == old) 0.00 : ffff800010cc4a6c: ldr x0, [x3, #16] 0.00 : ffff800010cc4a70: cmp x2, x0 0.00 : ffff800010cc4a74: b.eq ffff800010cc4af8 // b.none : __write_once_size(): 0.00 : ffff800010cc4a78: str x5, [x3, #8] : rb_erase(): : } 0.00 : ffff800010cc4a7c: ret : __write_once_size(): 20.88 : ffff800010cc4a80: str x5, [x2, #16] : __rb_erase_augmented(): : if (child) { 0.00 : ffff800010cc4a84: cbnz x5, ffff800010cc482c 15.95 : ffff800010cc4a88: b ffff800010cc49d8 : __write_once_size(): 0.00 : ffff800010cc4a8c: str x4, [x2, #16] : rb_erase(): 0.00 : ffff800010cc4a90: ret : __write_once_size(): 0.00 : ffff800010cc4a94: str x0, [x4, #16] : ____rb_erase_color(): : sibling = tmp1; 0.00 : ffff800010cc4a98: mov x0, x3 : __write_once_size(): 0.00 : ffff800010cc4a9c: ldr x3, [x3, #8] 0.00 : ffff800010cc4aa0: b ffff800010cc4930 : rb_set_black(): : rb->__rb_parent_color |= RB_BLACK; 0.00 : ffff800010cc4aa4: orr x0, x0, #0x1 0.00 : ffff800010cc4aa8: str x0, [x2] : rb_erase(): : } 0.00 : ffff800010cc4aac: ret : ____rb_erase_color(): : tmp1 = tmp2->rb_right; 0.00 : ffff800010cc4ab0: ldr x4, [x3, #8] : WRITE_ONCE(parent->rb_right, tmp2); 0.00 : ffff800010cc4ab4: mov x6, x3 : __write_once_size(): 0.00 : ffff800010cc4ab8: str x4, [x0, #16] 0.00 : ffff800010cc4abc: str x0, [x3, #8] 0.00 : ffff800010cc4ac0: str x3, [x2, #8] : ____rb_erase_color(): : if (tmp1) 0.00 : ffff800010cc4ac4: cbz x4, ffff800010cc4ad0 : rb_set_parent_color(): : rb->__rb_parent_color = (unsigned long)p | color; 0.00 : ffff800010cc4ac8: orr x5, x0, #0x1 0.00 : ffff800010cc4acc: str x5, [x4] 0.00 : ffff800010cc4ad0: mov x4, x0 0.00 : ffff800010cc4ad4: mov x0, x3 0.00 : ffff800010cc4ad8: ldr x3, [x3, #16] 0.00 : ffff800010cc4adc: b ffff800010cc49e4 : __write_once_size(): 0.00 : ffff800010cc4ae0: str x6, [x1] : rb_erase(): : } 0.00 : ffff800010cc4ae4: ret : __write_once_size(): 0.00 : ffff800010cc4ae8: str x5, [x1] : rb_erase(): 0.00 : ffff800010cc4aec: ret : __write_once_size(): 0.00 : ffff800010cc4af0: str x6, [x3, #16] : rb_erase(): 0.00 : ffff800010cc4af4: ret : __write_once_size(): 0.00 : ffff800010cc4af8: str x5, [x3, #16] : rb_erase(): 0.00 : ffff800010cc4afc: ret Percent | Source code & Disassembly of vmlinux for cycles (161 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b9b8 : : tick_nohz_next_event(): : { : return local_softirq_pending() & BIT(TIMER_SOFTIRQ); : } : : static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) : { 0.59 : ffff80001017b9b8: stp x29, x30, [sp, #-80]! 0.00 : ffff80001017b9bc: adrp x2, ffff800011907000 0.00 : ffff80001017b9c0: adrp x4, ffff800011b22000 0.00 : ffff80001017b9c4: mov x29, sp 0.59 : ffff80001017b9c8: stp x19, x20, [sp, #16] 0.00 : ffff80001017b9cc: adrp x20, ffff800011909000 0.57 : ffff80001017b9d0: stp x21, x22, [sp, #32] 0.00 : ffff80001017b9d4: add x3, x20, #0x908 0.00 : ffff80001017b9d8: str x23, [sp, #48] 0.00 : ffff80001017b9dc: mov x21, x0 0.00 : ffff80001017b9e0: mov w23, w1 0.64 : ffff80001017b9e4: ldr x0, [x3] 0.00 : ffff80001017b9e8: str x0, [x29, #72] 0.00 : ffff80001017b9ec: mov x0, #0x0 // #0 0.00 : ffff80001017b9f0: adrp x3, ffff800011907000 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001017b9f4: add x0, x2, #0x9c0 3.80 : ffff80001017b9f8: ldr w1, [x0] : __read_seqcount_begin(): : { : unsigned ret; : : repeat: : ret = READ_ONCE(s->sequence); : if (unlikely(ret & 1)) { 0.00 : ffff80001017b9fc: tbnz w1, #0, ffff80001017bb24 : raw_read_seqcount_begin(): : * section is tested by checking read_seqcount_retry function. : */ : static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) : { : unsigned ret = __read_seqcount_begin(s); : smp_rmb(); 3.17 : ffff80001017ba00: dmb ishld : tick_nohz_next_event(): : unsigned int seq; : : /* Read jiffies and the time when jiffies were updated last */ : do { : seq = read_seqbegin(&jiffies_lock); : basemono = last_jiffies_update; 33.83 : ffff80001017ba04: ldr x19, [x4, #1328] : basejiff = jiffies; 0.63 : ffff80001017ba08: ldr x22, [x3, #2432] : read_seqcount_retry(): : * If the critical section was invalid, it must be ignored (and typically : * retried). : */ : static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) : { : smp_rmb(); 0.00 : ffff80001017ba0c: dmb ishld : tick_nohz_next_event(): : } while (read_seqretry(&jiffies_lock, seq)); 26.63 : ffff80001017ba10: ldr w2, [x0] 0.00 : ffff80001017ba14: cmp w2, w1 0.00 : ffff80001017ba18: b.ne ffff80001017b9f8 // b.any : ts->last_jiffies = basejiff; 0.00 : ffff80001017ba1c: str x22, [x21, #160] : * because there is an already expired timer, so it will request : * immeditate expiry, which rearms the hardware timer with a : * minimal delta which brings us back to this place : * immediately. Lather, rinse and repeat... : */ : if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() || 0.00 : ffff80001017ba20: add x1, x29, #0x40 : ts->timer_expires_base = basemono; 0.00 : ffff80001017ba24: str x19, [x21, #176] : if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() || 0.00 : ffff80001017ba28: mov x0, x19 0.00 : ffff80001017ba2c: bl ffff800010162478 2.50 : ffff80001017ba30: cbz w0, ffff80001017bab0 : irq_work_needs_cpu() || local_timer_softirq_pending()) { : next_tick = basemono + TICK_NSEC; 0.00 : ffff80001017ba34: add x22, x19, #0x3d0, lsl #12 0.00 : ffff80001017ba38: add x22, x22, #0x900 : if (delta <= (u64)TICK_NSEC) { : /* : * Tell the timer code that the base is not idle, i.e. undo : * the effect of get_next_timer_interrupt(): : */ : timer_clear_idle(); 0.00 : ffff80001017ba3c: bl ffff80001016a9b0 : /* : * We've not stopped the tick yet, and there's a timer in the : * next period, so no point in stopping it either, bail. : */ : if (!ts->tick_stopped) { 0.60 : ffff80001017ba40: ldrb w0, [x21, #76] 0.00 : ffff80001017ba44: tbz w0, #1, ffff80001017bb04 : /* : * If this CPU is the one which had the do_timer() duty last, we limit : * the sleep time to the timekeeping max_deferment value. : * Otherwise we can sleep as long as we want. : */ : delta = timekeeping_max_deferment(); 0.00 : ffff80001017ba48: bl ffff80001016e578 : if (cpu != tick_do_timer_cpu && 0.00 : ffff80001017ba4c: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001017ba50: ldr w1, [x1, #712] 0.00 : ffff80001017ba54: cmp w1, w23 0.00 : ffff80001017ba58: b.eq ffff80001017ba68 // b.none 4.46 : ffff80001017ba5c: cmn w1, #0x1 0.00 : ffff80001017ba60: b.eq ffff80001017bb10 // b.none : (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last)) : delta = KTIME_MAX; 0.00 : ffff80001017ba64: mov x0, #0x7fffffffffffffff // #9223372036854775807 : : /* Calculate the next expiry time */ : if (delta < (KTIME_MAX - basemono)) 0.00 : ffff80001017ba68: mov x1, #0x7fffffffffffffff // #9223372036854775807 0.00 : ffff80001017ba6c: sub x2, x1, x19 : expires = basemono + delta; 0.00 : ffff80001017ba70: cmp x2, x0 0.00 : ffff80001017ba74: add x0, x0, x19 0.00 : ffff80001017ba78: csel x0, x0, x1, hi // hi = pmore : else : expires = KTIME_MAX; : : ts->timer_expires = min_t(u64, expires, next_tick); 0.00 : ffff80001017ba7c: cmp x0, x22 0.00 : ffff80001017ba80: csel x0, x0, x22, ls // ls = plast 0.00 : ffff80001017ba84: str x0, [x21, #168] : : out: : return ts->timer_expires; : } 0.00 : ffff80001017ba88: add x20, x20, #0x908 3.33 : ffff80001017ba8c: ldr x2, [x29, #72] 0.00 : ffff80001017ba90: ldr x1, [x20] 0.00 : ffff80001017ba94: eor x1, x2, x1 0.00 : ffff80001017ba98: cbnz x1, ffff80001017bb2c 0.00 : ffff80001017ba9c: ldp x19, x20, [sp, #16] 0.00 : ffff80001017baa0: ldp x21, x22, [sp, #32] 5.00 : ffff80001017baa4: ldr x23, [sp, #48] 0.00 : ffff80001017baa8: ldp x29, x30, [sp], #80 0.00 : ffff80001017baac: ret : irq_work_needs_cpu() || local_timer_softirq_pending()) { 0.65 : ffff80001017bab0: bl ffff8000101b4820 : if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() || 1.88 : ffff80001017bab4: tst w0, #0xff 0.00 : ffff80001017bab8: b.ne ffff80001017ba34 // b.any : local_timer_softirq_pending(): : return local_softirq_pending() & BIT(TIMER_SOFTIRQ); 6.66 : ffff80001017babc: adrp x0, ffff800011528000 0.00 : ffff80001017bac0: add x0, x0, #0x7c0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001017bac4: mrs x1, tpidr_el1 : tick_nohz_next_event(): : irq_work_needs_cpu() || local_timer_softirq_pending()) { 0.00 : ffff80001017bac8: ldr w0, [x0, x1] 0.00 : ffff80001017bacc: tbnz w0, #1, ffff80001017ba34 : next_tmr = get_next_timer_interrupt(basejiff, basemono); 1.29 : ffff80001017bad0: mov x1, x19 0.00 : ffff80001017bad4: mov x0, x22 0.00 : ffff80001017bad8: bl ffff80001016a838 : next_tick = next_rcu < next_tmr ? next_rcu : next_tmr; 0.00 : ffff80001017badc: ldr x22, [x29, #64] : ts->next_timer = next_tmr; 1.90 : ffff80001017bae0: str x0, [x21, #184] : if (delta <= (u64)TICK_NSEC) { 0.00 : ffff80001017bae4: mov x1, #0x900 // #2304 : next_tick = next_rcu < next_tmr ? next_rcu : next_tmr; 0.00 : ffff80001017bae8: cmp x22, x0 : if (delta <= (u64)TICK_NSEC) { 0.00 : ffff80001017baec: movk x1, #0x3d, lsl #16 : next_tick = next_rcu < next_tmr ? next_rcu : next_tmr; 1.25 : ffff80001017baf0: csel x22, x22, x0, ls // ls = plast : if (delta <= (u64)TICK_NSEC) { 0.00 : ffff80001017baf4: sub x0, x22, x19 0.00 : ffff80001017baf8: cmp x0, x1 0.00 : ffff80001017bafc: b.hi ffff80001017ba48 // b.pmore 0.00 : ffff80001017bb00: b ffff80001017ba3c : goto out; 0.00 : ffff80001017bb04: mov x0, #0x0 // #0 : ts->timer_expires = 0; 0.00 : ffff80001017bb08: str xzr, [x21, #168] : goto out; 0.00 : ffff80001017bb0c: b ffff80001017ba88 : (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last)) 0.00 : ffff80001017bb10: ldrb w2, [x21, #76] : delta = KTIME_MAX; 0.00 : ffff80001017bb14: mov x1, #0x7fffffffffffffff // #9223372036854775807 0.00 : ffff80001017bb18: tst x2, #0x8 0.00 : ffff80001017bb1c: csel x0, x0, x1, ne // ne = any 0.00 : ffff80001017bb20: b ffff80001017ba68 : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff80001017bb24: yield 0.00 : ffff80001017bb28: b ffff80001017b9f8 : tick_nohz_next_event(): : } 0.00 : ffff80001017bb2c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (153 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016bb90 : : hrtimer_get_next_event(): : * hrtimer_get_next_event - get the time until next expiry event : * : * Returns the next expiry time or KTIME_MAX if no timer is pending. : */ : u64 hrtimer_get_next_event(void) : { 0.00 : ffff80001016bb90: stp x29, x30, [sp, #-48]! 0.00 : ffff80001016bb94: mov x29, sp 14.43 : ffff80001016bb98: stp x19, x20, [sp, #16] : struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016bb9c: adrp x19, ffff800011525000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 3.33 : ffff80001016bba0: mrs x0, tpidr_el1 : hrtimer_get_next_event(): 0.00 : ffff80001016bba4: add x19, x19, #0x980 : { 0.00 : ffff80001016bba8: str x21, [sp, #32] : struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016bbac: add x19, x19, x0 0.00 : ffff80001016bbb0: mov x20, #0x7fffffffffffffff // #9223372036854775807 : u64 expires = KTIME_MAX; : unsigned long flags; : : raw_spin_lock_irqsave(&cpu_base->lock, flags); 0.00 : ffff80001016bbb4: mov x0, x19 7.67 : ffff80001016bbb8: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff80001016bbbc: mov x21, x0 : __hrtimer_hres_active(): : cpu_base->hres_active : 0; 19.36 : ffff80001016bbc0: ldrb w0, [x19, #16] : hrtimer_get_next_event(): : : if (!__hrtimer_hres_active(cpu_base)) 0.00 : ffff80001016bbc4: tbnz w0, #0, ffff80001016bbd8 : expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL); 0.00 : ffff80001016bbc8: mov w1, #0xff // #255 0.00 : ffff80001016bbcc: mov x0, x19 0.00 : ffff80001016bbd0: bl ffff80001016b020 <__hrtimer_get_next_event> 0.00 : ffff80001016bbd4: mov x20, x0 : : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 54.54 : ffff80001016bbd8: mov x1, x21 0.00 : ffff80001016bbdc: mov x0, x19 0.00 : ffff80001016bbe0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : : return expires; : } 0.67 : ffff80001016bbe4: ldr x21, [sp, #32] 0.00 : ffff80001016bbe8: mov x0, x20 0.00 : ffff80001016bbec: ldp x19, x20, [sp, #16] 0.00 : ffff80001016bbf0: ldp x29, x30, [sp], #48 0.00 : ffff80001016bbf4: ret Percent | Source code & Disassembly of vmlinux for cycles (156 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc8b58 : : timerqueue_del(): : * : * Removes the timer node from the timerqueue. Returns true if the queue is : * not empty after the remove. : */ : bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node) : { 2.61 : ffff800010cc8b58: stp x29, x30, [sp, #-32]! 0.00 : ffff800010cc8b5c: mov x29, sp 1.93 : ffff800010cc8b60: stp x19, x20, [sp, #16] 0.00 : ffff800010cc8b64: mov x20, x0 0.00 : ffff800010cc8b68: mov x19, x1 : WARN_ON_ONCE(RB_EMPTY_NODE(&node->node)); 3.30 : ffff800010cc8b6c: ldr x0, [x1] 0.00 : ffff800010cc8b70: cmp x0, x1 0.00 : ffff800010cc8b74: b.eq ffff800010cc8bbc // b.none : rb_erase_cached(): : } : : static inline void rb_erase_cached(struct rb_node *node, : struct rb_root_cached *root) : { : if (root->rb_leftmost == node) 3.26 : ffff800010cc8b78: ldr x0, [x20, #8] 0.00 : ffff800010cc8b7c: cmp x19, x0 1.24 : ffff800010cc8b80: b.eq ffff800010cc8bac // b.none : root->rb_leftmost = rb_next(node); : rb_erase(node, &root->rb_root); 14.38 : ffff800010cc8b84: mov x0, x19 0.00 : ffff800010cc8b88: mov x1, x20 0.00 : ffff800010cc8b8c: bl ffff800010cc4750 : timerqueue_del(): : : rb_erase_cached(&node->node, &head->rb_root); : RB_CLEAR_NODE(&node->node); 0.66 : ffff800010cc8b90: str x19, [x19] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 15.27 : ffff800010cc8b94: ldr x0, [x20] : timerqueue_del(): : : return !RB_EMPTY_ROOT(&head->rb_root.rb_root); : } 29.22 : ffff800010cc8b98: ldp x19, x20, [sp, #16] : return !RB_EMPTY_ROOT(&head->rb_root.rb_root); 0.00 : ffff800010cc8b9c: cmp x0, #0x0 : } 0.00 : ffff800010cc8ba0: cset w0, ne // ne = any 18.40 : ffff800010cc8ba4: ldp x29, x30, [sp], #32 0.00 : ffff800010cc8ba8: ret : rb_erase_cached(): : root->rb_leftmost = rb_next(node); 5.33 : ffff800010cc8bac: mov x0, x19 0.00 : ffff800010cc8bb0: bl ffff800010cc5050 4.41 : ffff800010cc8bb4: str x0, [x20, #8] 0.00 : ffff800010cc8bb8: b ffff800010cc8b84 : timerqueue_del(): : WARN_ON_ONCE(RB_EMPTY_NODE(&node->node)); 0.00 : ffff800010cc8bbc: brk #0x800 0.00 : ffff800010cc8bc0: b ffff800010cc8b78 Percent | Source code & Disassembly of vmlinux for cycles (283 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001009dc80 : : hw_breakpoint_thread_switch(): : : /* : * Context-switcher for restoring suspended breakpoints. : */ : void hw_breakpoint_thread_switch(struct task_struct *next) : { 0.00 : ffff80001009dc80: stp x29, x30, [sp, #-32]! 0.00 : ffff80001009dc84: mov x29, sp 6.36 : ffff80001009dc88: stp x19, x20, [sp, #16] : : current_debug_info = ¤t->thread.debug; : next_debug_info = &next->thread.debug; : : /* Update breakpoints. */ : if (current_debug_info->bps_disabled != next_debug_info->bps_disabled) 0.00 : ffff80001009dc8c: add x20, x0, #0xc78 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 1.84 : ffff80001009dc90: mrs x19, sp_el0 : hw_breakpoint_thread_switch(): 0.00 : ffff80001009dc94: add x19, x19, #0xc78 0.00 : ffff80001009dc98: ldr w1, [x19, #4] 77.83 : ffff80001009dc9c: ldr w0, [x20, #4] 0.00 : ffff80001009dca0: cmp w1, w0 0.00 : ffff80001009dca4: b.eq ffff80001009dcbc // b.none : toggle_bp_registers(AARCH64_DBG_REG_BCR, 0.00 : ffff80001009dca8: cmp w0, #0x0 0.00 : ffff80001009dcac: mov w1, #0x0 // #0 0.00 : ffff80001009dcb0: cset w2, eq // eq = none 0.00 : ffff80001009dcb4: mov w0, #0x10 // #16 0.00 : ffff80001009dcb8: bl ffff80001009cbf0 : DBG_ACTIVE_EL0, : !next_debug_info->bps_disabled); : : /* Update watchpoints. */ : if (current_debug_info->wps_disabled != next_debug_info->wps_disabled) 0.00 : ffff80001009dcbc: ldr w1, [x19, #8] 5.34 : ffff80001009dcc0: ldr w0, [x20, #8] 0.00 : ffff80001009dcc4: cmp w1, w0 0.00 : ffff80001009dcc8: b.eq ffff80001009dce0 // b.none : toggle_bp_registers(AARCH64_DBG_REG_WCR, 0.00 : ffff80001009dccc: cmp w0, #0x0 0.00 : ffff80001009dcd0: mov w1, #0x0 // #0 0.00 : ffff80001009dcd4: cset w2, eq // eq = none 0.00 : ffff80001009dcd8: mov w0, #0x30 // #48 0.00 : ffff80001009dcdc: bl ffff80001009cbf0 : DBG_ACTIVE_EL0, : !next_debug_info->wps_disabled); : } 1.55 : ffff80001009dce0: ldp x19, x20, [sp, #16] 7.09 : ffff80001009dce4: ldp x29, x30, [sp], #32 0.00 : ffff80001009dce8: ret Percent | Source code & Disassembly of vmlinux for cycles (163 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc5050 : : rb_next(): : : struct rb_node *rb_next(const struct rb_node *node) : { : struct rb_node *parent; : : if (RB_EMPTY_NODE(node)) 12.33 : ffff800010cc5050: ldr x1, [x0] : { 0.00 : ffff800010cc5054: mov x2, x0 : if (RB_EMPTY_NODE(node)) 0.00 : ffff800010cc5058: cmp x0, x1 0.00 : ffff800010cc505c: b.eq ffff800010cc509c // b.none : : /* : * If we have a right-hand child, go down and then left as far : * as we can. : */ : if (node->rb_right) { 32.75 : ffff800010cc5060: ldr x0, [x0, #8] 0.00 : ffff800010cc5064: cbnz x0, ffff800010cc5070 3.04 : ffff800010cc5068: b ffff800010cc5090 0.00 : ffff800010cc506c: mov x0, x1 : node = node->rb_right; : while (node->rb_left) 0.00 : ffff800010cc5070: ldr x1, [x0, #16] 0.00 : ffff800010cc5074: cbnz x1, ffff800010cc506c : */ : while ((parent = rb_parent(node)) && node == parent->rb_right) : node = parent; : : return parent; : } 40.73 : ffff800010cc5078: ret : while ((parent = rb_parent(node)) && node == parent->rb_right) 1.83 : ffff800010cc507c: ldr x1, [x0, #8] 1.89 : ffff800010cc5080: cmp x2, x1 0.00 : ffff800010cc5084: b.ne ffff800010cc5078 // b.any 0.00 : ffff800010cc5088: ldr x1, [x0] 0.00 : ffff800010cc508c: mov x2, x0 7.43 : ffff800010cc5090: ands x0, x1, #0xfffffffffffffffc 0.00 : ffff800010cc5094: b.ne ffff800010cc507c // b.any : } 0.00 : ffff800010cc5098: ret : return NULL; 0.00 : ffff800010cc509c: mov x0, #0x0 // #0 : } 0.00 : ffff800010cc50a0: ret Percent | Source code & Disassembly of vmlinux for cycles (131 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010114450 <__task_rq_lock>: : __task_rq_lock(): : /* : * __task_rq_lock - lock the rq @p resides on. : */ : struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) : __acquires(rq->lock) : { 5.24 : ffff800010114450: stp x29, x30, [sp, #-64]! 0.00 : ffff800010114454: mov x29, sp 27.38 : ffff800010114458: stp x21, x22, [sp, #32] 0.00 : ffff80001011445c: adrp x21, ffff800011909000 3.81 : ffff800010114460: str x23, [sp, #48] 0.00 : ffff800010114464: mov x22, x0 0.00 : ffff800010114468: adrp x23, ffff800011528000 : struct rq *rq; : : lockdep_assert_held(&p->pi_lock); : : for (;;) { : rq = task_rq(p); 0.00 : ffff80001011446c: add x21, x21, #0x928 1.48 : ffff800010114470: add x23, x23, #0xe80 : { 0.00 : ffff800010114474: stp x19, x20, [sp, #16] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.76 : ffff800010114478: ldr w0, [x22, #68] : __task_rq_lock(): : rq = task_rq(p); 0.00 : ffff80001011447c: mov x19, x23 2.23 : ffff800010114480: ldr x20, [x21, w0, uxtw #3] 0.00 : ffff800010114484: add x20, x19, x20 : raw_spin_lock(&rq->lock); 0.00 : ffff800010114488: mov x0, x20 0.00 : ffff80001011448c: bl ffff800010cd80f0 <_raw_spin_lock> : __read_once_size(): 8.42 : ffff800010114490: ldr w0, [x22, #68] : __task_rq_lock(): : if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 40.07 : ffff800010114494: ldr x0, [x21, w0, uxtw #3] 0.00 : ffff800010114498: add x19, x19, x0 0.00 : ffff80001011449c: cmp x20, x19 0.00 : ffff8000101144a0: b.ne ffff8000101144c8 <__task_rq_lock+0x78> // b.any : __read_once_size(): 2.28 : ffff8000101144a4: ldr w0, [x22, #104] : __task_rq_lock(): 0.00 : ffff8000101144a8: cmp w0, #0x2 0.00 : ffff8000101144ac: b.eq ffff8000101144c8 <__task_rq_lock+0x78> // b.none : raw_spin_unlock(&rq->lock); : : while (unlikely(task_on_rq_migrating(p))) : cpu_relax(); : } : } 1.59 : ffff8000101144b0: mov x0, x20 0.00 : ffff8000101144b4: ldr x23, [sp, #48] 0.00 : ffff8000101144b8: ldp x19, x20, [sp, #16] 0.71 : ffff8000101144bc: ldp x21, x22, [sp, #32] 6.05 : ffff8000101144c0: ldp x29, x30, [sp], #64 0.00 : ffff8000101144c4: ret : raw_spin_unlock(&rq->lock); 0.00 : ffff8000101144c8: mov x0, x20 0.00 : ffff8000101144cc: bl ffff800010cd7c38 <_raw_spin_unlock> : __read_once_size(): 0.00 : ffff8000101144d0: ldr w0, [x22, #104] : __task_rq_lock(): : while (unlikely(task_on_rq_migrating(p))) 0.00 : ffff8000101144d4: cmp w0, #0x2 0.00 : ffff8000101144d8: b.ne ffff800010114478 <__task_rq_lock+0x28> // b.any : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff8000101144dc: yield : __read_once_size(): 0.00 : ffff8000101144e0: ldr w0, [x22, #104] : __task_rq_lock(): 0.00 : ffff8000101144e4: cmp w0, #0x2 0.00 : ffff8000101144e8: b.ne ffff800010114478 <__task_rq_lock+0x28> // b.any 0.00 : ffff8000101144ec: b ffff8000101144dc <__task_rq_lock+0x8c> Percent | Source code & Disassembly of vmlinux for cycles (127 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010251990 <___slab_alloc>: : ___slab_alloc(): : * Version of __slab_alloc to use when we know that interrupts are : * already disabled (which is the case for bulk allocation). : */ : static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, : unsigned long addr, struct kmem_cache_cpu *c) : { 0.00 : ffff800010251990: stp x29, x30, [sp, #-224]! 0.00 : ffff800010251994: mov x29, sp 0.00 : ffff800010251998: stp x26, x27, [sp, #72] 0.00 : ffff80001025199c: mov x26, x4 0.00 : ffff8000102519a0: str x19, [sp, #16] 0.00 : ffff8000102519a4: adrp x19, ffff800011909000 0.00 : ffff8000102519a8: stp x21, x22, [sp, #32] 0.00 : ffff8000102519ac: add x4, x19, #0x908 0.00 : ffff8000102519b0: str x28, [sp, #88] 0.00 : ffff8000102519b4: mov x27, x0 0.00 : ffff8000102519b8: str x3, [x29, #136] 0.00 : ffff8000102519bc: mov w22, w1 : void *freelist; : struct page *page; : : page = c->page; 0.00 : ffff8000102519c0: ldr x28, [x26, #16] : { 0.00 : ffff8000102519c4: ldr x0, [x4] 0.00 : ffff8000102519c8: str x0, [x29, #216] 0.00 : ffff8000102519cc: mov x0, #0x0 // #0 0.00 : ffff8000102519d0: mov w21, w2 : if (!page) 0.00 : ffff8000102519d4: cbz x28, ffff800010251b38 <___slab_alloc+0x1a8> : node_match(): : if (node != NUMA_NO_NODE && page_to_nid(page) != node) 0.00 : ffff8000102519d8: cmn w21, #0x1 0.00 : ffff8000102519dc: b.eq ffff800010251ad0 <___slab_alloc+0x140> // b.none : page_to_nid(): : #else : static inline int page_to_nid(const struct page *page) : { : struct page *p = (struct page *)page; : : return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK; 0.00 : ffff8000102519e0: ldr x0, [x28] 0.00 : ffff8000102519e4: lsr x0, x0, #62 : node_match(): 0.00 : ffff8000102519e8: cmp w21, w0 0.00 : ffff8000102519ec: b.eq ffff800010251ad0 <___slab_alloc+0x140> // b.none 0.00 : ffff8000102519f0: str x20, [x29, #24] : ___slab_alloc(): : * By rights, we should be searching for a slab page that was : * PFMEMALLOC but right now, we are losing the pfmemalloc : * information when the page leaves the per-cpu allocator : */ : if (unlikely(!pfmemalloc_match(page, gfpflags))) { : deactivate_slab(s, page, c->freelist, c); 0.00 : ffff8000102519f4: add x4, x26, #0x10 0.00 : ffff8000102519f8: stp x23, x24, [x29, #48] 0.00 : ffff8000102519fc: mov x3, x26 0.00 : ffff800010251a00: str x25, [x29, #64] 0.00 : ffff800010251a04: mov x1, x28 0.00 : ffff800010251a08: mov x0, x27 0.00 : ffff800010251a0c: ldr x2, [x26] 0.00 : ffff800010251a10: bl ffff80001024fd40 : gfp_zonelist(): : */ : : static inline int gfp_zonelist(gfp_t flags) : { : #ifdef CONFIG_NUMA : if (unlikely(flags & __GFP_THISNODE)) 0.00 : ffff800010251a14: ubfx x0, x22, #21, #1 : gfp_zone(): : z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) & 0.00 : ffff800010251a18: ubfiz w1, w22, #1, #4 0.00 : ffff800010251a1c: mov w20, #0x122 // #290 : get_node(): : : }; : : static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) : { : return s->node[node]; 0.00 : ffff800010251a20: sxtw x23, w21 : node_zonelist(): : * For the normal case of non-DISCONTIGMEM systems the NODE_DATA() gets : * optimized to &contig_page_data at compile-time. : */ : static inline struct zonelist *node_zonelist(int nid, gfp_t flags) : { : return NODE_DATA(nid)->node_zonelists + gfp_zonelist(flags); 0.00 : ffff800010251a24: lsl x24, x0, #4 : gfp_zone(): : z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) & 0.00 : ffff800010251a28: movk w20, #0x132, lsl #16 : node_zonelist(): : return NODE_DATA(nid)->node_zonelists + gfp_zonelist(flags); 0.00 : ffff800010251a2c: add x24, x24, x0 : gfp_zone(): : z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) & 0.00 : ffff800010251a30: asr w20, w20, w1 : get_node(): 0.00 : ffff800010251a34: add x23, x23, #0x30 : gfp_zone(): 0.00 : ffff800010251a38: and w20, w20, #0x3 : node_zonelist(): : return NODE_DATA(nid)->node_zonelists + gfp_zonelist(flags); 0.00 : ffff800010251a3c: lsl x0, x24, #4 : kmem_cache_debug(): : return unlikely(s->flags & SLAB_DEBUG_FLAGS); 0.00 : ffff800010251a40: mov w24, #0xd00 // #3328 : numa_node_id(): : : #ifndef numa_node_id : /* Returns the number of the current Node. */ : static inline int numa_node_id(void) : { : return raw_cpu_read(numa_node); 0.00 : ffff800010251a44: adrp x25, ffff800011526000 : node_zonelist(): 0.00 : ffff800010251a48: str x0, [x29, #112] : kmem_cache_debug(): 0.00 : ffff800010251a4c: movk w24, #0x21, lsl #16 : numa_node_id(): 0.00 : ffff800010251a50: add x0, x25, #0xd58 0.00 : ffff800010251a54: str x0, [x29, #120] : ___slab_alloc(): : c->tid = next_tid(c->tid); : return freelist; : : new_slab: : : if (slub_percpu_partial(c)) { 0.00 : ffff800010251a58: ldr x28, [x26, #24] 0.00 : ffff800010251a5c: cbnz x28, ffff800010251f6c <___slab_alloc+0x5dc> : new_slab_objects(): : WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO)); 0.00 : ffff800010251a60: ldr x0, [x27, #64] 0.00 : ffff800010251a64: cbz x0, ffff800010251a70 <___slab_alloc+0xe0> 0.00 : ffff800010251a68: tbz w22, #8, ffff800010251a70 <___slab_alloc+0xe0> 0.00 : ffff800010251a6c: brk #0x800 0.00 : ffff800010251a70: add x28, x26, #0x10 : get_partial(): : if (node == NUMA_NO_NODE) 0.00 : ffff800010251a74: cmn w21, #0x1 0.00 : ffff800010251a78: b.eq ffff800010251b48 <___slab_alloc+0x1b8> // b.none : object = get_partial_node(s, get_node(s, searchnode), c, flags); 0.00 : ffff800010251a7c: ldr x1, [x27, x23, lsl #3] 0.00 : ffff800010251a80: mov w3, w22 0.00 : ffff800010251a84: mov x2, x28 0.00 : ffff800010251a88: mov x0, x27 0.00 : ffff800010251a8c: bl ffff800010251618 0.00 : ffff800010251a90: mov x5, x0 : new_slab_objects(): : if (freelist) 0.00 : ffff800010251a94: cbz x0, ffff800010251dd8 <___slab_alloc+0x448> 0.00 : ffff800010251a98: ldr x28, [x26, #16] : kmem_cache_debug(): : return unlikely(s->flags & SLAB_DEBUG_FLAGS); 0.00 : ffff800010251a9c: ldr w0, [x27, #8] : ___slab_alloc(): : slab_out_of_memory(s, gfpflags, node); : return NULL; : } : : page = c->page; : if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) 0.00 : ffff800010251aa0: tst w0, w24 0.00 : ffff800010251aa4: b.ne ffff800010251f18 <___slab_alloc+0x588> // b.any : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010251aa8: ldr x1, [x28, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff800010251aac: sub x0, x1, #0x1 0.00 : ffff800010251ab0: tst x1, #0x1 0.00 : ffff800010251ab4: csel x0, x0, x28, ne // ne = any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010251ab8: ldr x0, [x0] : pfmemalloc_match(): : if (unlikely(PageSlabPfmemalloc(page))) 0.00 : ffff800010251abc: tbnz w0, #5, ffff800010251ef4 <___slab_alloc+0x564> 0.00 : ffff800010251ac0: ldr x20, [x29, #24] 0.00 : ffff800010251ac4: ldp x23, x24, [x29, #48] 0.00 : ffff800010251ac8: ldr x25, [x29, #64] 0.00 : ffff800010251acc: b ffff800010251af4 <___slab_alloc+0x164> : __read_once_size(): 0.00 : ffff800010251ad0: ldr x1, [x28, #8] : compound_head(): 0.00 : ffff800010251ad4: sub x0, x1, #0x1 0.00 : ffff800010251ad8: tst x1, #0x1 0.00 : ffff800010251adc: csel x0, x0, x28, ne // ne = any : test_bit(): 37.05 : ffff800010251ae0: ldr x0, [x0] : pfmemalloc_match(): 0.00 : ffff800010251ae4: tst w0, #0x20 0.00 : ffff800010251ae8: b.ne ffff800010251dc0 <___slab_alloc+0x430> // b.any : ___slab_alloc(): : freelist = c->freelist; 3.94 : ffff800010251aec: ldr x5, [x26] : if (freelist) 0.00 : ffff800010251af0: cbz x5, ffff800010251c30 <___slab_alloc+0x2a0> : get_freepointer(): : return freelist_dereference(s, object + s->offset); 0.00 : ffff800010251af4: ldr w1, [x27, #32] : next_tid(): : return tid + TID_STEP; 0.00 : ffff800010251af8: ldr x0, [x26, #8] 0.00 : ffff800010251afc: add x0, x0, #0x100 : ___slab_alloc(): : c->freelist = get_freepointer(s, freelist); 0.00 : ffff800010251b00: ldr x1, [x5, x1] : c->tid = next_tid(c->tid); 0.00 : ffff800010251b04: stp x1, x0, [x26] : !alloc_debug_processing(s, page, freelist, addr)) : goto new_slab; /* Slab failed checks. Next slab needed */ : : deactivate_slab(s, page, get_freepointer(s, freelist), c); : return freelist; : } 0.00 : ffff800010251b08: add x19, x19, #0x908 0.00 : ffff800010251b0c: mov x0, x5 0.00 : ffff800010251b10: ldr x2, [x29, #216] 0.78 : ffff800010251b14: ldr x1, [x19] 0.00 : ffff800010251b18: eor x1, x2, x1 0.00 : ffff800010251b1c: cbnz x1, ffff800010251f88 <___slab_alloc+0x5f8> 0.00 : ffff800010251b20: ldr x19, [sp, #16] 0.00 : ffff800010251b24: ldp x21, x22, [sp, #32] 0.00 : ffff800010251b28: ldp x26, x27, [sp, #72] 0.00 : ffff800010251b2c: ldr x28, [sp, #88] 0.00 : ffff800010251b30: ldp x29, x30, [sp], #224 0.00 : ffff800010251b34: ret 0.00 : ffff800010251b38: str x20, [x29, #24] 0.00 : ffff800010251b3c: stp x23, x24, [x29, #48] 0.00 : ffff800010251b40: str x25, [x29, #64] 0.00 : ffff800010251b44: b ffff800010251a14 <___slab_alloc+0x84> : numa_node_id(): 0.00 : ffff800010251b48: bl ffff80001024c6e8 <__my_cpu_offset> 0.00 : ffff800010251b4c: ldr x1, [x29, #120] : get_partial(): : object = get_partial_node(s, get_node(s, searchnode), c, flags); 0.00 : ffff800010251b50: mov w3, w22 : get_node(): 0.00 : ffff800010251b54: ldrsw x1, [x1, x0] : get_partial(): 0.00 : ffff800010251b58: mov x2, x28 0.00 : ffff800010251b5c: mov x0, x27 : get_node(): 0.00 : ffff800010251b60: add x1, x1, #0x30 : get_partial(): 0.00 : ffff800010251b64: ldr x1, [x27, x1, lsl #3] 0.00 : ffff800010251b68: bl ffff800010251618 0.00 : ffff800010251b6c: mov x5, x0 : if (object || node != NUMA_NO_NODE) 0.00 : ffff800010251b70: cbnz x0, ffff800010251a98 <___slab_alloc+0x108> : get_any_partial(): : if (!s->remote_node_defrag_ratio || 0.00 : ffff800010251b74: ldr w0, [x27, #368] 0.00 : ffff800010251b78: cbz w0, ffff800010251dd8 <___slab_alloc+0x448> : get_cycles() % 1024 > s->remote_node_defrag_ratio) 0.00 : ffff800010251b7c: adrp x0, ffff800011ac4000 0.00 : ffff800010251b80: ldr x0, [x0, #3984] 0.00 : ffff800010251b84: blr x0 0.00 : ffff800010251b88: and x0, x0, #0x3ff 0.00 : ffff800010251b8c: ldr w1, [x27, #368] : if (!s->remote_node_defrag_ratio || 0.00 : ffff800010251b90: cmp x0, x1 0.00 : ffff800010251b94: b.hi ffff800010251dd8 <___slab_alloc+0x448> // b.pmore 0.00 : ffff800010251b98: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> : node_zonelist(): 0.00 : ffff800010251b9c: add x0, x0, #0x128 0.00 : ffff800010251ba0: str x0, [x29, #104] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010251ba4: nop : read_mems_allowed_begin(): : * read_mems_allowed_retry prevents these artificial failures. : */ : static inline unsigned int read_mems_allowed_begin(void) : { : if (!static_branch_unlikely(&cpusets_pre_enable_key)) : return 0; 0.00 : ffff800010251ba8: str wzr, [x29, #100] : get_any_partial(): : zonelist = node_zonelist(mempolicy_slab_node(), flags); 0.00 : ffff800010251bac: bl ffff800010244c20 : node_zonelist(): 0.00 : ffff800010251bb0: ldr x1, [x29, #104] 0.00 : ffff800010251bb4: ldr x4, [x1, w0, sxtw #3] 0.00 : ffff800010251bb8: mov x0, #0x1a00 // #6656 0.00 : ffff800010251bbc: add x4, x4, x0 0.00 : ffff800010251bc0: ldr x0, [x29, #112] 0.00 : ffff800010251bc4: add x4, x4, x0 : next_zones_zonelist(): : */ : static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, : enum zone_type highest_zoneidx, : nodemask_t *nodes) : { : if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) 0.00 : ffff800010251bc8: ldr w0, [x4, #8] 0.00 : ffff800010251bcc: cmp w20, w0 0.00 : ffff800010251bd0: b.cc ffff800010251c18 <___slab_alloc+0x288> // b.lo, b.ul, b.last 0.00 : ffff800010251bd4: nop : get_any_partial(): : for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 0.00 : ffff800010251bd8: ldr x0, [x4] 0.00 : ffff800010251bdc: cbz x0, ffff800010251dd4 <___slab_alloc+0x444> : n = get_node(s, zone_to_nid(zone)); 0.00 : ffff800010251be0: ldr w0, [x0, #72] : get_node(): 0.00 : ffff800010251be4: add x1, x27, w0, sxtw #3 0.00 : ffff800010251be8: ldr x25, [x1, #384] : get_any_partial(): : if (n && cpuset_zone_allowed(zone, flags) && 0.00 : ffff800010251bec: cbz x25, ffff800010251c08 <___slab_alloc+0x278> : arch_static_branch(): 0.00 : ffff800010251bf0: nop : get_any_partial(): 0.00 : ffff800010251bf4: ldr x1, [x25, #8] 0.00 : ffff800010251bf8: ldr x0, [x27, #16] 0.00 : ffff800010251bfc: cmp x1, x0 0.00 : ffff800010251c00: b.hi ffff800010251eac <___slab_alloc+0x51c> // b.pmore 0.00 : ffff800010251c04: nop : next_zones_zonelist(): 0.00 : ffff800010251c08: ldr w0, [x4, #24] : get_any_partial(): : for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 0.00 : ffff800010251c0c: add x4, x4, #0x10 : next_zones_zonelist(): 0.00 : ffff800010251c10: cmp w20, w0 0.00 : ffff800010251c14: b.cs ffff800010251bd8 <___slab_alloc+0x248> // b.hs, b.nlast : return z; : return __next_zones_zonelist(z, highest_zoneidx, nodes); 0.00 : ffff800010251c18: mov x0, x4 0.00 : ffff800010251c1c: mov x2, #0x0 // #0 0.00 : ffff800010251c20: mov w1, w20 0.00 : ffff800010251c24: bl ffff8000101f3cf0 <__next_zones_zonelist> 0.00 : ffff800010251c28: mov x4, x0 0.00 : ffff800010251c2c: b ffff800010251bd8 <___slab_alloc+0x248> 0.00 : ffff800010251c30: str x25, [x29, #64] : __cmpxchg_double_slab(): : if (cmpxchg_double(&page->freelist, &page->counters, 0.00 : ffff800010251c34: add x25, x28, #0x20 0.00 : ffff800010251c38: str x20, [x29, #24] : __ll_sc__cmpxchg_double_mb(): : \ : return ret; \ : } : : __CMPXCHG_DBL( , , , ) : __CMPXCHG_DBL(_mb, dmb ish, l, "memory") 0.00 : ffff800010251c3c: mov x20, #0x0 // #0 0.00 : ffff800010251c40: stp x23, x24, [x29, #48] : __lse_atomic64_fetch_or_acquire(): : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff800010251c44: mov x23, #0x1 // #1 : get_freelist(): : counters = page->counters; 0.00 : ffff800010251c48: ldp x5, x24, [x28, #32] : new.counters = counters; 0.82 : ffff800010251c4c: str x24, [x29, #184] : new.inuse = page->objects; 0.00 : ffff800010251c50: ldrh w0, [x28, #42] : new.frozen = freelist != NULL; 0.00 : ffff800010251c54: cmp x5, #0x0 : __cmpxchg_double_slab(): : if (s->flags & __CMPXCHG_DOUBLE) { 0.00 : ffff800010251c58: ldr w2, [x27, #8] : get_freelist(): : new.frozen = freelist != NULL; 0.00 : ffff800010251c5c: cset w3, ne // ne = any 0.00 : ffff800010251c60: lsr w1, w24, #24 : new.inuse = page->objects; 0.00 : ffff800010251c64: ubfx x0, x0, #0, #15 0.00 : ffff800010251c68: strh w0, [x29, #184] : new.frozen = freelist != NULL; 0.00 : ffff800010251c6c: bfi w1, w3, #7, #1 0.00 : ffff800010251c70: strb w1, [x29, #187] : } while (!__cmpxchg_double_slab(s, page, 0.00 : ffff800010251c74: ldr x3, [x29, #184] : __cmpxchg_double_slab(): : if (s->flags & __CMPXCHG_DOUBLE) { 0.00 : ffff800010251c78: tbz w2, #30, ffff800010251d20 <___slab_alloc+0x390> : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 3.18 : ffff800010251c7c: b ffff800010251d58 <___slab_alloc+0x3c8> 0.00 : ffff800010251c80: b ffff800010251d58 <___slab_alloc+0x3c8> : __lse__cmpxchg_double_mb(): : \ : return x0; \ : } : : __CMPXCHG_DBL( , ) : __CMPXCHG_DBL(_mb, al, "memory") 0.00 : ffff800010251c84: mov x2, #0x0 // #0 0.00 : ffff800010251c88: mov x0, x5 0.00 : ffff800010251c8c: mov x1, x24 0.00 : ffff800010251c90: mov x4, x25 0.00 : ffff800010251c94: caspal x0, x1, x2, x3, [x25] 54.24 : ffff800010251c98: eor x0, x0, x5 0.00 : ffff800010251c9c: eor x1, x1, x24 0.00 : ffff800010251ca0: orr x0, x0, x1 : __cmpxchg_double_slab(): : if (cmpxchg_double(&page->freelist, &page->counters, 0.00 : ffff800010251ca4: cbz x0, ffff800010251cf8 <___slab_alloc+0x368> : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff800010251ca8: yield 0.00 : ffff800010251cac: b ffff800010251c48 <___slab_alloc+0x2b8> : __cmpxchg_double_slab(): : if (page->freelist == freelist_old && 0.00 : ffff800010251cb0: ldr x0, [x28, #40] 0.00 : ffff800010251cb4: cmp x24, x0 0.00 : ffff800010251cb8: b.ne ffff800010251d84 <___slab_alloc+0x3f4> // b.any : __read_once_size(): 0.00 : ffff800010251cbc: ldr x0, [x28] : __cmpxchg_double_slab(): : page->counters = counters_new; 0.00 : ffff800010251cc0: stp xzr, x3, [x28, #32] : __clear_bit_unlock(): : { : unsigned long old; : : p += BIT_WORD(nr); : old = READ_ONCE(*p); : old &= ~BIT_MASK(nr); 0.00 : ffff800010251cc4: and x0, x0, #0xfffffffffffffffe : atomic64_set_release(): : : #ifndef atomic64_set_release : static inline void : atomic64_set_release(atomic64_t *v, s64 i) : { : smp_store_release(&(v)->counter, i); 0.00 : ffff800010251cc8: stlr x0, [x28] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010251ccc: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff800010251cd0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010251cd4: sub x0, x0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010251cd8: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010251cdc: cbz x0, ffff800010251ce8 <___slab_alloc+0x358> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010251ce0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010251ce4: cbnz x0, ffff800010251cf8 <___slab_alloc+0x368> 0.00 : ffff800010251ce8: str x5, [x29, #128] : __bit_spin_unlock(): : BUG_ON(!test_bit(bitnum, addr)); : #endif : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) : __clear_bit_unlock(bitnum, addr); : #endif : preempt_enable(); 0.00 : ffff800010251cec: bl ffff800010cd2e50 0.00 : ffff800010251cf0: ldr x5, [x29, #128] 0.00 : ffff800010251cf4: nop : ___slab_alloc(): : if (!freelist) { 0.00 : ffff800010251cf8: cbnz x5, ffff800010251ac0 <___slab_alloc+0x130> : c->page = NULL; 0.00 : ffff800010251cfc: str xzr, [x26, #16] 0.00 : ffff800010251d00: b ffff800010251a14 <___slab_alloc+0x84> 0.00 : ffff800010251d04: stp x5, x3, [x29, #120] : bit_spin_lock(): : preempt_enable(); 0.00 : ffff800010251d08: bl ffff800010cd2e50 0.00 : ffff800010251d0c: ldp x5, x3, [x29, #120] : cpu_relax(): 0.00 : ffff800010251d10: yield : test_bit(): 0.00 : ffff800010251d14: ldr x0, [x28] : bit_spin_lock(): : } while (test_bit(bitnum, addr)); 0.00 : ffff800010251d18: tbnz w0, #0, ffff800010251d10 <___slab_alloc+0x380> 0.00 : ffff800010251d1c: nop : get_current(): 0.00 : ffff800010251d20: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff800010251d24: ldr w0, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff800010251d28: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010251d2c: str w0, [x1, #16] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010251d30: ldr x0, [x28] : test_and_set_bit_lock(): : if (READ_ONCE(*p) & mask) 0.00 : ffff800010251d34: tbz w0, #0, ffff800010251d64 <___slab_alloc+0x3d4> : get_current(): 0.00 : ffff800010251d38: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff800010251d3c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010251d40: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010251d44: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010251d48: cbz x0, ffff800010251d04 <___slab_alloc+0x374> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010251d4c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010251d50: cbz x0, ffff800010251d04 <___slab_alloc+0x374> 0.00 : ffff800010251d54: b ffff800010251d10 <___slab_alloc+0x380> : __ll_sc__cmpxchg_double_mb(): 0.00 : ffff800010251d58: b ffff800010255ab4 : __cmpxchg_double_slab(): : if (cmpxchg_double(&page->freelist, &page->counters, 0.00 : ffff800010251d5c: cbnz x0, ffff800010251ca8 <___slab_alloc+0x318> 0.00 : ffff800010251d60: b ffff800010251cf8 <___slab_alloc+0x368> : arch_static_branch_jump(): 0.00 : ffff800010251d64: b ffff800010251db8 <___slab_alloc+0x428> 0.00 : ffff800010251d68: b ffff800010251db8 <___slab_alloc+0x428> : __lse_atomic64_fetch_or_acquire(): : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff800010251d6c: mov x0, x23 0.00 : ffff800010251d70: ldseta x0, x0, [x28] : bit_spin_lock(): : while (unlikely(test_and_set_bit_lock(bitnum, addr))) { 0.00 : ffff800010251d74: tbnz w0, #0, ffff800010251d38 <___slab_alloc+0x3a8> : __cmpxchg_double_slab(): : if (page->freelist == freelist_old && 0.00 : ffff800010251d78: ldr x0, [x28, #32] 0.00 : ffff800010251d7c: cmp x5, x0 0.00 : ffff800010251d80: b.eq ffff800010251cb0 <___slab_alloc+0x320> // b.none : __read_once_size(): 0.00 : ffff800010251d84: ldr x0, [x28] : __clear_bit_unlock(): : old &= ~BIT_MASK(nr); 0.00 : ffff800010251d88: and x0, x0, #0xfffffffffffffffe : atomic64_set_release(): 0.00 : ffff800010251d8c: stlr x0, [x28] : get_current(): 0.00 : ffff800010251d90: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff800010251d94: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010251d98: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010251d9c: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010251da0: cbz x0, ffff800010251dac <___slab_alloc+0x41c> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010251da4: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010251da8: cbnz x0, ffff800010251ca8 <___slab_alloc+0x318> : __bit_spin_unlock(): : preempt_enable(); 0.00 : ffff800010251dac: bl ffff800010cd2e50 : cpu_relax(): 0.00 : ffff800010251db0: yield 0.00 : ffff800010251db4: b ffff800010251c48 <___slab_alloc+0x2b8> : __ll_sc_atomic64_fetch_or_acquire(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010251db8: b ffff800010255adc 0.00 : ffff800010251dbc: b ffff800010251d74 <___slab_alloc+0x3e4> : pfmemalloc_match(): : return gfp_pfmemalloc_allowed(gfpflags); 0.00 : ffff800010251dc0: mov w0, w22 0.00 : ffff800010251dc4: bl ffff80001022aa30 : ___slab_alloc(): : if (unlikely(!pfmemalloc_match(page, gfpflags))) { 0.00 : ffff800010251dc8: tst w0, #0xff 0.00 : ffff800010251dcc: b.ne ffff800010251aec <___slab_alloc+0x15c> // b.any 0.00 : ffff800010251dd0: b ffff8000102519f0 <___slab_alloc+0x60> : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010251dd4: nop : new_slab_objects(): : page = new_slab(s, flags, node); 0.00 : ffff800010251dd8: mov w2, w21 0.00 : ffff800010251ddc: mov w1, w22 0.00 : ffff800010251de0: mov x0, x27 0.00 : ffff800010251de4: bl ffff800010251090 0.00 : ffff800010251de8: mov x28, x0 : if (page) { 0.00 : ffff800010251dec: cbz x0, ffff800010251e34 <___slab_alloc+0x4a4> : c = raw_cpu_ptr(s->cpu_slab); 0.00 : ffff800010251df0: ldr x2, [x27] 0.00 : ffff800010251df4: bl ffff80001024c6e8 <__my_cpu_offset> 0.00 : ffff800010251df8: add x26, x2, x0 : if (c->page) 0.00 : ffff800010251dfc: ldr x1, [x26, #16] 0.00 : ffff800010251e00: cbz x1, ffff800010251e24 <___slab_alloc+0x494> : flush_slab(): : deactivate_slab(s, c->page, c->freelist, c); 0.00 : ffff800010251e04: ldr x2, [x2, x0] 0.00 : ffff800010251e08: add x4, x26, #0x10 0.00 : ffff800010251e0c: mov x0, x27 0.00 : ffff800010251e10: mov x3, x26 0.00 : ffff800010251e14: bl ffff80001024fd40 : next_tid(): : return tid + TID_STEP; 0.00 : ffff800010251e18: ldr x0, [x26, #8] 0.00 : ffff800010251e1c: add x0, x0, #0x100 : flush_slab(): : c->tid = next_tid(c->tid); 0.00 : ffff800010251e20: str x0, [x26, #8] : new_slab_objects(): : freelist = page->freelist; 0.00 : ffff800010251e24: ldr x5, [x28, #32] : page->freelist = NULL; 0.00 : ffff800010251e28: str xzr, [x28, #32] : c->page = page; 0.00 : ffff800010251e2c: str x28, [x26, #16] : ___slab_alloc(): : if (unlikely(!freelist)) { 0.00 : ffff800010251e30: cbnz x5, ffff800010251a9c <___slab_alloc+0x10c> : return NULL; 0.00 : ffff800010251e34: mov x5, #0x0 // #0 : slab_out_of_memory(s, gfpflags, node); 0.00 : ffff800010251e38: mov w2, w21 : return NULL; 0.00 : ffff800010251e3c: str x5, [x29, #136] : slab_out_of_memory(s, gfpflags, node); 0.00 : ffff800010251e40: mov w1, w22 0.00 : ffff800010251e44: mov x0, x27 0.00 : ffff800010251e48: bl ffff80001024d538 : return NULL; 0.00 : ffff800010251e4c: ldr x20, [x29, #24] 0.00 : ffff800010251e50: ldp x23, x24, [x29, #48] 0.00 : ffff800010251e54: ldr x25, [x29, #64] 0.00 : ffff800010251e58: ldr x5, [x29, #136] 0.00 : ffff800010251e5c: b ffff800010251b08 <___slab_alloc+0x178> : get_current(): 0.00 : ffff800010251e60: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff800010251e64: ldr w1, [x0, #1984] 0.00 : ffff800010251e68: str w1, [x29, #100] : __read_seqcount_begin(): : { : unsigned ret; : : repeat: : ret = READ_ONCE(s->sequence); : if (unlikely(ret & 1)) { 0.00 : ffff800010251e6c: tbnz w1, #0, ffff800010251e78 <___slab_alloc+0x4e8> : raw_read_seqcount_begin(): : * section is tested by checking read_seqcount_retry function. : */ : static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) : { : unsigned ret = __read_seqcount_begin(s); : smp_rmb(); 0.00 : ffff800010251e70: dmb ishld 0.00 : ffff800010251e74: b ffff800010251bac <___slab_alloc+0x21c> : cpu_relax(): 0.00 : ffff800010251e78: yield : __read_once_size(): 0.00 : ffff800010251e7c: ldr w1, [x0, #1984] 0.00 : ffff800010251e80: str w1, [x29, #100] : __read_seqcount_begin(): : if (unlikely(ret & 1)) { 0.00 : ffff800010251e84: tbnz w1, #0, ffff800010251e78 <___slab_alloc+0x4e8> : raw_read_seqcount_begin(): : smp_rmb(); 0.00 : ffff800010251e88: dmb ishld 0.00 : ffff800010251e8c: b ffff800010251bac <___slab_alloc+0x21c> 0.00 : ffff800010251e90: str x4, [x29, #128] : __cpuset_zone_allowed(): : return __cpuset_node_allowed(zone_to_nid(z), gfp_mask); 0.00 : ffff800010251e94: mov w1, w22 0.00 : ffff800010251e98: bl ffff80001019f268 <__cpuset_node_allowed> : get_any_partial(): : if (n && cpuset_zone_allowed(zone, flags) && 0.00 : ffff800010251e9c: tst w0, #0xff 0.00 : ffff800010251ea0: ldr x4, [x29, #128] 0.00 : ffff800010251ea4: b.eq ffff800010251c08 <___slab_alloc+0x278> // b.none 0.00 : ffff800010251ea8: b ffff800010251bf4 <___slab_alloc+0x264> 0.00 : ffff800010251eac: str x4, [x29, #128] : object = get_partial_node(s, n, c, flags); 0.00 : ffff800010251eb0: mov w3, w22 0.00 : ffff800010251eb4: mov x2, x28 0.00 : ffff800010251eb8: mov x1, x25 0.00 : ffff800010251ebc: mov x0, x27 0.00 : ffff800010251ec0: bl ffff800010251618 : if (object) { 0.00 : ffff800010251ec4: ldr x4, [x29, #128] : object = get_partial_node(s, n, c, flags); 0.00 : ffff800010251ec8: mov x5, x0 : if (object) { 0.00 : ffff800010251ecc: cbz x0, ffff800010251c08 <___slab_alloc+0x278> 0.00 : ffff800010251ed0: b ffff800010251a98 <___slab_alloc+0x108> : get_current(): 0.00 : ffff800010251ed4: mrs x0, sp_el0 : read_seqcount_retry(): : * If the critical section was invalid, it must be ignored (and typically : * retried). : */ : static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) : { : smp_rmb(); 0.00 : ffff800010251ed8: dmb ishld : get_any_partial(): : } while (read_mems_allowed_retry(cpuset_mems_cookie)); 0.00 : ffff800010251edc: ldr w0, [x0, #1984] 0.00 : ffff800010251ee0: ldr w1, [x29, #100] 0.00 : ffff800010251ee4: cmp w0, w1 0.00 : ffff800010251ee8: b.eq ffff800010251dd8 <___slab_alloc+0x448> // b.none : arch_static_branch(): 0.00 : ffff800010251eec: nop 0.00 : ffff800010251ef0: b ffff800010251ba8 <___slab_alloc+0x218> 0.00 : ffff800010251ef4: str x5, [x29, #128] : pfmemalloc_match(): : return gfp_pfmemalloc_allowed(gfpflags); 0.00 : ffff800010251ef8: mov w0, w22 0.00 : ffff800010251efc: bl ffff80001022aa30 : ___slab_alloc(): : if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) 0.00 : ffff800010251f00: tst w0, #0xff 0.00 : ffff800010251f04: ldr x5, [x29, #128] 0.00 : ffff800010251f08: b.ne ffff800010251ac0 <___slab_alloc+0x130> // b.any : kmem_cache_debug(): : return unlikely(s->flags & SLAB_DEBUG_FLAGS); 0.00 : ffff800010251f0c: ldr w0, [x27, #8] : ___slab_alloc(): : if (kmem_cache_debug(s) && 0.00 : ffff800010251f10: tst w0, w24 0.00 : ffff800010251f14: b.eq ffff800010251f38 <___slab_alloc+0x5a8> // b.none : !alloc_debug_processing(s, page, freelist, addr)) 0.00 : ffff800010251f18: ldr x3, [x29, #136] 0.00 : ffff800010251f1c: mov x2, x5 0.00 : ffff800010251f20: str x5, [x29, #128] 0.00 : ffff800010251f24: mov x1, x28 0.00 : ffff800010251f28: mov x0, x27 0.00 : ffff800010251f2c: bl ffff80001024f318 : if (kmem_cache_debug(s) && 0.00 : ffff800010251f30: ldr x5, [x29, #128] 0.00 : ffff800010251f34: cbz w0, ffff800010251a58 <___slab_alloc+0xc8> : get_freepointer(): : return freelist_dereference(s, object + s->offset); 0.00 : ffff800010251f38: ldr w2, [x27, #32] : ___slab_alloc(): : deactivate_slab(s, page, get_freepointer(s, freelist), c); 0.00 : ffff800010251f3c: add x4, x26, #0x10 0.00 : ffff800010251f40: str x5, [x29, #136] 0.00 : ffff800010251f44: mov x3, x26 0.00 : ffff800010251f48: mov x1, x28 0.00 : ffff800010251f4c: mov x0, x27 0.00 : ffff800010251f50: ldr x2, [x5, x2] 0.00 : ffff800010251f54: bl ffff80001024fd40 : return freelist; 0.00 : ffff800010251f58: ldr x20, [x29, #24] 0.00 : ffff800010251f5c: ldp x23, x24, [x29, #48] 0.00 : ffff800010251f60: ldr x25, [x29, #64] 0.00 : ffff800010251f64: ldr x5, [x29, #136] 0.00 : ffff800010251f68: b ffff800010251b08 <___slab_alloc+0x178> : page = c->page = slub_percpu_partial(c); 0.00 : ffff800010251f6c: str x28, [x26, #16] : slub_set_percpu_partial(c, page); 0.00 : ffff800010251f70: ldr x0, [x28, #8] 0.00 : ffff800010251f74: str x0, [x26, #24] 0.00 : ffff800010251f78: ldr x20, [x29, #24] 0.00 : ffff800010251f7c: ldp x23, x24, [x29, #48] 0.00 : ffff800010251f80: ldr x25, [x29, #64] 0.00 : ffff800010251f84: b ffff8000102519d8 <___slab_alloc+0x48> 0.00 : ffff800010251f88: str x20, [x29, #24] 0.00 : ffff800010251f8c: stp x23, x24, [x29, #48] 0.00 : ffff800010251f90: str x25, [x29, #64] : } 0.00 : ffff800010251f94: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (253 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010087c20 : : uao_thread_switch(): : : /* Restore the UAO state depending on next's addr_limit */ : void uao_thread_switch(struct task_struct *next) : { : if (IS_ENABLED(CONFIG_ARM64_UAO)) { : if (task_thread_info(next)->addr_limit == KERNEL_DS) 3.63 : ffff800010087c20: ldr x0, [x0, #8] 0.00 : ffff800010087c24: cmn x0, #0x1 0.00 : ffff800010087c28: b.eq ffff800010087c34 // b.none : asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO)); : else : asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO)); 0.00 : ffff800010087c2c: nop : } : } 5.16 : ffff800010087c30: ret : asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO)); 3.84 : ffff800010087c34: nop : } 87.36 : ffff800010087c38: ret Percent | Source code & Disassembly of vmlinux for cycles (138 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010081000 <__do_softirq>: : __do_softirq(): : static inline bool lockdep_softirq_start(void) { return false; } : static inline void lockdep_softirq_end(bool in_hardirq) { } : #endif : : asmlinkage __visible void __softirq_entry __do_softirq(void) : { 0.71 : ffff800010081000: stp x29, x30, [sp, #-144]! : unsigned long end = jiffies + MAX_SOFTIRQ_TIME; 0.00 : ffff800010081004: adrp x1, ffff800011907000 : { 0.00 : ffff800010081008: mov x29, sp 3.03 : ffff80001008100c: stp x19, x20, [sp, #16] 0.00 : ffff800010081010: stp x21, x22, [sp, #32] : * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC : * again if the socket is related to swapping. : */ : current->flags &= ~PF_MEMALLOC; : : pending = local_softirq_pending(); 0.00 : ffff800010081014: adrp x21, ffff800011528000 : { 0.00 : ffff800010081018: stp x23, x24, [sp, #48] : pending = local_softirq_pending(); 0.00 : ffff80001008101c: add x21, x21, #0x7c0 : { 0.74 : ffff800010081020: stp x25, x26, [sp, #64] 0.00 : ffff800010081024: stp x27, x28, [sp, #80] : unsigned long end = jiffies + MAX_SOFTIRQ_TIME; 0.00 : ffff800010081028: ldr x24, [x1, #2432] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 5.82 : ffff80001008102c: mrs x19, sp_el0 : __do_softirq(): : unsigned long old_flags = current->flags; 0.00 : ffff800010081030: ldr w0, [x19, #44] : unsigned long end = jiffies + MAX_SOFTIRQ_TIME; 0.00 : ffff800010081034: add x2, x24, #0x1 : unsigned long old_flags = current->flags; 0.00 : ffff800010081038: str w0, [x29, #128] : current->flags &= ~PF_MEMALLOC; 0.00 : ffff80001008103c: and w0, w0, #0xfffff7ff 0.00 : ffff800010081040: str w0, [x19, #44] : unsigned long end = jiffies + MAX_SOFTIRQ_TIME; 2.18 : ffff800010081044: str x2, [x29, #112] : pending = local_softirq_pending(); 0.00 : ffff800010081048: mov x0, x21 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001008104c: mrs x1, tpidr_el1 : __do_softirq(): 0.00 : ffff800010081050: ldr w24, [x0, x1] : account_irq_enter_time(): : #endif : : static inline void account_irq_enter_time(struct task_struct *tsk) : { : vtime_account_irq_enter(tsk); : irqtime_account_irq(tsk); 0.00 : ffff800010081054: mov x0, x19 0.00 : ffff800010081058: bl ffff80001011b180 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.64 : ffff80001008105c: ldr w0, [x19, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010081060: add w0, w0, #0x100 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010081064: str w0, [x19, #16] : __do_softirq(): : : trace_softirq_entry(vec_nr); : h->action(h); : trace_softirq_exit(vec_nr); : if (unlikely(prev_count != preempt_count())) { : pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", 0.00 : ffff800010081068: adrp x26, ffff800010d05000 0.00 : ffff80001008106c: add x0, x26, #0xa50 0.00 : ffff800010081070: str x0, [x29, #104] : __local_bh_disable_ip(): : extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); : #else : static __always_inline void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) : { : preempt_count_add(cnt); : barrier(); 0.00 : ffff800010081074: mov w0, #0xa // #10 0.00 : ffff800010081078: str w0, [x29, #132] : __do_softirq(): : set_softirq_pending(0); 0.69 : ffff80001008107c: str x21, [x29, #136] : __my_cpu_offset(): 0.00 : ffff800010081080: mrs x1, tpidr_el1 : __do_softirq(): 0.00 : ffff800010081084: ldr x0, [x29, #136] 0.69 : ffff800010081088: str wzr, [x0, x1] : arch_local_irq_enable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff80001008108c: mov x0, #0xe0 // #224 0.00 : ffff800010081090: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 14.53 : ffff800010081094: nop : ffs(): : * the libc and compiler builtin ffs routines, therefore : * differs in spirit from the above ffz (man ffs). : */ : static __always_inline int ffs(int x) : { : return __builtin_ffs(x); 0.00 : ffff800010081098: rbit w27, w24 0.00 : ffff80001008109c: cmp w24, #0x0 0.00 : ffff8000100810a0: clz w27, w27 0.00 : ffff8000100810a4: csinc w27, wzr, w27, eq // eq = none : __do_softirq(): : while ((softirq_bit = ffs(pending))) { 0.00 : ffff8000100810a8: cbz w27, ffff800010081138 <__do_softirq+0x138> 0.00 : ffff8000100810ac: adrp x19, ffff800011906000 0.00 : ffff8000100810b0: add x19, x19, #0xc0 0.00 : ffff8000100810b4: adrp x20, ffff80001151f000 0.00 : ffff8000100810b8: mov x26, x19 : kstat_incr_softirqs_this_cpu(): : extern unsigned int kstat_irqs_cpu(unsigned int irq, int cpu); : extern void kstat_incr_irq_this_cpu(unsigned int irq); : : static inline void kstat_incr_softirqs_this_cpu(unsigned int irq) : { : __this_cpu_inc(kstat.softirqs[irq]); 0.63 : ffff8000100810bc: add x20, x20, #0x10 : __do_softirq(): : pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", 0.00 : ffff8000100810c0: adrp x22, ffff8000111c2000 0.00 : ffff8000100810c4: add x0, x22, #0x0 0.00 : ffff8000100810c8: str x0, [x29, #120] 0.00 : ffff8000100810cc: nop : h += softirq_bit - 1; 8.79 : ffff8000100810d0: sbfiz x28, x27, #3, #32 : vec_nr, softirq_to_name[vec_nr], h->action, : prev_count, preempt_count()); : preempt_count_set(prev_count); : } : h++; : pending >>= softirq_bit; 0.00 : ffff8000100810d4: lsr w24, w24, w27 : h += softirq_bit - 1; 0.00 : ffff8000100810d8: sub x21, x28, #0x8 : ffs(): 0.00 : ffff8000100810dc: rbit w27, w24 : __do_softirq(): 0.00 : ffff8000100810e0: add x0, x26, x21 : ffs(): 0.00 : ffff8000100810e4: clz w27, w27 : __do_softirq(): : vec_nr = h - softirq_vec; 0.00 : ffff8000100810e8: sub x1, x0, x19 : __my_cpu_offset(): 0.00 : ffff8000100810ec: mrs x5, tpidr_el1 : __do_softirq(): 0.00 : ffff8000100810f0: asr x23, x1, #3 : get_current(): 0.00 : ffff8000100810f4: mrs x22, sp_el0 : kstat_incr_softirqs_this_cpu(): 0.00 : ffff8000100810f8: ubfiz x2, x23, #2, #32 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000100810fc: ldr w25, [x22, #16] : kstat_incr_softirqs_this_cpu(): 0.00 : ffff800010081100: add x2, x2, #0x8 0.00 : ffff800010081104: add x2, x2, x20 0.74 : ffff800010081108: ldr w3, [x2, x5] 0.00 : ffff80001008110c: add w3, w3, #0x1 3.59 : ffff800010081110: str w3, [x2, x5] : __do_softirq(): : h->action(h); 0.00 : ffff800010081114: ldr x2, [x26, x21] 0.00 : ffff800010081118: blr x2 : __read_once_size(): 0.00 : ffff80001008111c: ldr w0, [x22, #16] : __do_softirq(): : if (unlikely(prev_count != preempt_count())) { 0.00 : ffff800010081120: cmp w0, w25 0.00 : ffff800010081124: b.ne ffff800010081204 <__do_softirq+0x204> // b.any : ffs(): 0.00 : ffff800010081128: cmp w24, #0x0 : __do_softirq(): : h++; 0.00 : ffff80001008112c: add x26, x26, x28 : ffs(): 0.00 : ffff800010081130: csinc w27, wzr, w27, eq // eq = none : __do_softirq(): : while ((softirq_bit = ffs(pending))) { 0.00 : ffff800010081134: cbnz w27, ffff8000100810d0 <__do_softirq+0xd0> : } : : if (__this_cpu_read(ksoftirqd) == current) 1.48 : ffff800010081138: adrp x0, ffff80001151e000 0.00 : ffff80001008113c: add x0, x0, #0xf80 : __my_cpu_offset(): 0.00 : ffff800010081140: mrs x1, tpidr_el1 : __do_softirq(): 0.00 : ffff800010081144: ldr x1, [x0, x1] : get_current(): 2.18 : ffff800010081148: mrs x0, sp_el0 : __do_softirq(): 0.00 : ffff80001008114c: cmp x1, x0 0.00 : ffff800010081150: b.eq ffff800010081234 <__do_softirq+0x234> // b.none : arch_local_irq_disable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff800010081154: mov x0, #0x60 // #96 0.00 : ffff800010081158: msr daifset, #0x2 : __my_cpu_offset(): 10.19 : ffff80001008115c: mrs x1, tpidr_el1 : __do_softirq(): : rcu_softirq_qs(); : local_irq_disable(); : : pending = local_softirq_pending(); 1.49 : ffff800010081160: ldr x0, [x29, #136] 0.71 : ffff800010081164: ldr w24, [x0, x1] : if (pending) { 0.00 : ffff800010081168: cbz w24, ffff8000100811a0 <__do_softirq+0x1a0> : if (time_before(jiffies, end) && !need_resched() && 0.00 : ffff80001008116c: adrp x0, ffff800011907000 0.00 : ffff800010081170: ldr x1, [x29, #112] 0.63 : ffff800010081174: ldr x0, [x0, #2432] 0.00 : ffff800010081178: sub x0, x0, x1 0.00 : ffff80001008117c: tbz x0, #63, ffff80001008119c <__do_softirq+0x19c> : get_current(): 0.00 : ffff800010081180: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010081184: ldr x0, [x0] : __do_softirq(): 0.00 : ffff800010081188: tbnz w0, #1, ffff80001008119c <__do_softirq+0x19c> 0.00 : ffff80001008118c: ldr w0, [x29, #132] 0.00 : ffff800010081190: subs w0, w0, #0x1 0.00 : ffff800010081194: str w0, [x29, #132] 0.00 : ffff800010081198: b.ne ffff800010081080 <__do_softirq+0x80> // b.any : --max_restart) : goto restart; : : wakeup_softirqd(); 0.00 : ffff80001008119c: bl ffff8000100ec5c0 : get_current(): 1.98 : ffff8000100811a0: mrs x19, sp_el0 : account_irq_exit_time(): : } : : static inline void account_irq_exit_time(struct task_struct *tsk) : { : vtime_account_irq_exit(tsk); : irqtime_account_irq(tsk); 1.46 : ffff8000100811a4: mov x0, x19 0.00 : ffff8000100811a8: bl ffff80001011b180 : __read_once_size(): 0.00 : ffff8000100811ac: ldr w0, [x19, #16] 0.00 : ffff8000100811b0: ldr w0, [x19, #16] 0.00 : ffff8000100811b4: ldr w0, [x19, #16] : __preempt_count_sub(): : } : : static inline void __preempt_count_sub(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc -= val; 0.00 : ffff8000100811b8: sub w0, w0, #0x100 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000100811bc: str w0, [x19, #16] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000100811c0: ldr w0, [x19, #16] : __do_softirq(): : } : : lockdep_softirq_end(in_hardirq); : account_irq_exit_time(current); : __local_bh_enable(SOFTIRQ_OFFSET); : WARN_ON_ONCE(in_interrupt()); 0.00 : ffff8000100811c4: tst w0, #0x1fff00 0.00 : ffff8000100811c8: b.ne ffff80001008123c <__do_softirq+0x23c> // b.any : current_restore_flags(): : : static inline void : current_restore_flags(unsigned long orig_flags, unsigned long flags) : { : current->flags &= ~flags; : current->flags |= orig_flags & flags; 0.73 : ffff8000100811cc: ldr w1, [x29, #128] : get_current(): 0.74 : ffff8000100811d0: mrs x2, sp_el0 : current_restore_flags(): : current->flags &= ~flags; 0.00 : ffff8000100811d4: ldr w0, [x2, #44] : current->flags |= orig_flags & flags; 0.00 : ffff8000100811d8: and w1, w1, #0x800 : current->flags &= ~flags; 0.00 : ffff8000100811dc: and w0, w0, #0xfffff7ff : current->flags |= orig_flags & flags; 0.00 : ffff8000100811e0: orr w0, w0, w1 0.00 : ffff8000100811e4: str w0, [x2, #44] : __do_softirq(): : current_restore_flags(old_flags, PF_MEMALLOC); : } 0.00 : ffff8000100811e8: ldp x19, x20, [sp, #16] 0.00 : ffff8000100811ec: ldp x21, x22, [sp, #32] 0.00 : ffff8000100811f0: ldp x23, x24, [sp, #48] 0.73 : ffff8000100811f4: ldp x25, x26, [sp, #64] 0.00 : ffff8000100811f8: ldp x27, x28, [sp, #80] 0.00 : ffff8000100811fc: ldp x29, x30, [sp], #144 0.00 : ffff800010081200: ret : pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", 0.00 : ffff800010081204: ldr x0, [x29, #104] 0.00 : ffff800010081208: mov w4, w25 0.00 : ffff80001008120c: ldr x3, [x26, x21] 0.00 : ffff800010081210: mov w1, w23 : __read_once_size(): 0.00 : ffff800010081214: ldr w5, [x22, #16] : __do_softirq(): 0.00 : ffff800010081218: ldr x2, [x0, w23, uxtw #3] 0.00 : ffff80001008121c: ldr x0, [x29, #120] 0.00 : ffff800010081220: bl ffff80001014a85c : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010081224: str w25, [x22, #16] 0.00 : ffff800010081228: b ffff800010081128 <__do_softirq+0x128> : arch_local_irq_enable(): : pmr_sync(); 0.00 : ffff80001008122c: dsb sy 34.88 : ffff800010081230: b ffff800010081098 <__do_softirq+0x98> : __do_softirq(): : rcu_softirq_qs(); 0.00 : ffff800010081234: bl ffff800010160ea8 0.00 : ffff800010081238: b ffff800010081154 <__do_softirq+0x154> : WARN_ON_ONCE(in_interrupt()); 0.00 : ffff80001008123c: brk #0x800 0.00 : ffff800010081240: b ffff8000100811cc <__do_softirq+0x1cc> Percent | Source code & Disassembly of vmlinux for cycles (134 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012ae08 : : put_prev_task_rt(): : set_next_task_rt(rq, p, true); : return p; : } : : static void put_prev_task_rt(struct rq *rq, struct task_struct *p) : { 19.40 : ffff80001012ae08: stp x29, x30, [sp, #-48]! 0.00 : ffff80001012ae0c: mov x29, sp 30.59 : ffff80001012ae10: stp x19, x20, [sp, #16] 0.00 : ffff80001012ae14: mov x20, x0 0.00 : ffff80001012ae18: mov x19, x1 : update_curr_rt(rq); 0.00 : ffff80001012ae1c: bl ffff80001012aab8 : : update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1); 3.80 : ffff80001012ae20: mov w2, #0x1 // #1 0.00 : ffff80001012ae24: mov x1, x20 : rq_clock_pelt(): : static inline u64 rq_clock_pelt(struct rq *rq) : { : lockdep_assert_held(&rq->lock); : assert_clock_updated(rq); : : return rq->clock_pelt - rq->lost_idle_time; 2.99 : ffff80001012ae28: ldr x3, [x20, #2440] 11.72 : ffff80001012ae2c: ldr x0, [x20, #2448] : put_prev_task_rt(): 0.00 : ffff80001012ae30: sub x0, x3, x0 0.00 : ffff80001012ae34: bl ffff800010134c40 : : /* : * The previous task needs to be made eligible for pushing : * if it is still active : */ : if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1) 2.93 : ffff80001012ae38: ldrh w0, [x19, #484] 0.00 : ffff80001012ae3c: cbz w0, ffff80001012aea8 0.00 : ffff80001012ae40: ldr w0, [x19, #732] 0.00 : ffff80001012ae44: cmp w0, #0x1 0.00 : ffff80001012ae48: b.le ffff80001012aea8 0.00 : ffff80001012ae4c: stp x21, x22, [x29, #32] : enqueue_pushable_task(): : plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 0.00 : ffff80001012ae50: add x21, x19, #0x378 0.00 : ffff80001012ae54: add x22, x20, #0x878 0.00 : ffff80001012ae58: mov x0, x21 0.00 : ffff80001012ae5c: mov x1, x22 0.00 : ffff80001012ae60: bl ffff800010cc2748 : plist_node_init(): : * @node: &struct plist_node pointer : * @prio: initial node priority : */ : static inline void plist_node_init(struct plist_node *node, int prio) : { : node->prio = prio; 0.00 : ffff80001012ae64: ldr w2, [x19, #108] : INIT_LIST_HEAD(&node->prio_list); 0.00 : ffff80001012ae68: add x1, x19, #0x380 : node->prio = prio; 0.00 : ffff80001012ae6c: str w2, [x19, #888] : INIT_LIST_HEAD(&node->node_list); 0.00 : ffff80001012ae70: add x2, x19, #0x390 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012ae74: str x1, [x19, #896] : enqueue_pushable_task(): : plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks); 0.00 : ffff80001012ae78: mov x0, x21 : INIT_LIST_HEAD(): : * the result is an empty list. : */ : static inline void INIT_LIST_HEAD(struct list_head *list) : { : WRITE_ONCE(list->next, list); : list->prev = list; 0.00 : ffff80001012ae7c: str x1, [x19, #904] : enqueue_pushable_task(): 0.00 : ffff80001012ae80: mov x1, x22 : __write_once_size(): 0.00 : ffff80001012ae84: str x2, [x19, #912] : INIT_LIST_HEAD(): 0.00 : ffff80001012ae88: str x2, [x19, #920] : enqueue_pushable_task(): 0.00 : ffff80001012ae8c: bl ffff800010cc2670 : if (p->prio < rq->rt.highest_prio.next) 0.00 : ffff80001012ae90: ldr w0, [x19, #108] 0.00 : ffff80001012ae94: ldr w1, [x20, #2140] 0.00 : ffff80001012ae98: cmp w0, w1 0.00 : ffff80001012ae9c: b.ge ffff80001012aeb4 // b.tcont : rq->rt.highest_prio.next = p->prio; 0.00 : ffff80001012aea0: str w0, [x20, #2140] 0.00 : ffff80001012aea4: ldp x21, x22, [x29, #32] : put_prev_task_rt(): : enqueue_pushable_task(rq, p); : } 6.05 : ffff80001012aea8: ldp x19, x20, [sp, #16] 22.50 : ffff80001012aeac: ldp x29, x30, [sp], #48 0.00 : ffff80001012aeb0: ret 0.00 : ffff80001012aeb4: ldp x21, x22, [x29, #32] 0.00 : ffff80001012aeb8: ldp x19, x20, [sp, #16] 0.00 : ffff80001012aebc: ldp x29, x30, [sp], #48 0.00 : ffff80001012aec0: ret Percent | Source code & Disassembly of vmlinux for cycles (131 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010127cf8 <__pick_next_task_fair>: : __pick_next_task_fair(): : : return NULL; : } : : static struct task_struct *__pick_next_task_fair(struct rq *rq) : { 35.09 : ffff800010127cf8: stp x29, x30, [sp, #-16]! : return pick_next_task_fair(rq, NULL, NULL); 0.00 : ffff800010127cfc: mov x2, #0x0 // #0 58.00 : ffff800010127d00: mov x1, #0x0 // #0 : { 0.00 : ffff800010127d04: mov x29, sp : return pick_next_task_fair(rq, NULL, NULL); 2.36 : ffff800010127d08: bl ffff800010127a00 : } 4.55 : ffff800010127d0c: ldp x29, x30, [sp], #16 0.00 : ffff800010127d10: ret Percent | Source code & Disassembly of vmlinux for cycles (350 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010207308 : : get_user_pages_fast(): : * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns : * -errno. : */ : int get_user_pages_fast(unsigned long start, int nr_pages, : unsigned int gup_flags, struct page **pages) : { 0.29 : ffff800010207308: stp x29, x30, [sp, #-80]! 0.00 : ffff80001020730c: mov x29, sp 0.00 : ffff800010207310: str x19, [sp, #16] 0.00 : ffff800010207314: adrp x19, ffff800011909000 0.00 : ffff800010207318: stp x21, x22, [sp, #32] 0.00 : ffff80001020731c: add x4, x19, #0x908 0.00 : ffff800010207320: mov w21, w2 0.00 : ffff800010207324: ldr x5, [x4] 2.27 : ffff800010207328: str x5, [x29, #72] 0.00 : ffff80001020732c: mov x5, #0x0 // #0 : /* : * FOLL_PIN must only be set internally by the pin_user_pages*() APIs, : * never directly by the caller, so enforce that: : */ : if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) 0.00 : ffff800010207330: tbnz w21, #18, ffff800010207438 0.00 : ffff800010207334: str x23, [x29, #48] 0.00 : ffff800010207338: mov w23, w1 : internal_get_user_pages_fast(): : int nr = 0, ret = 0; 0.00 : ffff80001020733c: str wzr, [x29, #68] : if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | 0.00 : ffff800010207340: mov w1, #0xffee // #65518 0.00 : ffff800010207344: movk w1, #0xfffa, lsl #16 0.00 : ffff800010207348: tst w2, w1 0.00 : ffff80001020734c: b.ne ffff800010207498 // b.any 0.58 : ffff800010207350: str x20, [x29, #24] : sign_extend64(): : * @index: 0 based bit index (0<=index<64) to sign bit : */ : static inline __s64 sign_extend64(__u64 value, int index) : { : __u8 shift = 63 - index; : return (__s64)(value << shift) >> shift; 0.00 : ffff800010207354: sbfx x20, x0, #0, #56 : internal_get_user_pages_fast(): : start = untagged_addr(start) & PAGE_MASK; 0.00 : ffff800010207358: and x0, x0, #0xfffffffffffff000 : len = (unsigned long) nr_pages << PAGE_SHIFT; 0.00 : ffff80001020735c: sbfiz x2, x23, #12, #32 : start = untagged_addr(start) & PAGE_MASK; 2.61 : ffff800010207360: and x20, x20, x0 : return 0; 0.00 : ffff800010207364: mov w22, #0x0 // #0 : end = start + len; 0.00 : ffff800010207368: add x1, x20, x2 : if (end <= start) 0.00 : ffff80001020736c: cmp x20, x1 0.00 : ffff800010207370: b.cc ffff8000102073a4 // b.lo, b.ul, b.last 0.00 : ffff800010207374: ldr x20, [x29, #24] 0.00 : ffff800010207378: ldr x23, [x29, #48] : get_user_pages_fast(): : return -EINVAL; : : return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); : } 0.00 : ffff80001020737c: add x19, x19, #0x908 0.00 : ffff800010207380: mov w0, w22 0.00 : ffff800010207384: ldr x2, [x29, #72] 0.00 : ffff800010207388: ldr x1, [x19] 0.00 : ffff80001020738c: eor x1, x2, x1 0.00 : ffff800010207390: cbnz x1, ffff8000102074fc 0.55 : ffff800010207394: ldr x19, [sp, #16] 0.87 : ffff800010207398: ldp x21, x22, [sp, #32] 0.00 : ffff80001020739c: ldp x29, x30, [sp], #80 0.00 : ffff8000102073a0: ret 0.29 : ffff8000102073a4: str x24, [x29, #56] 0.00 : ffff8000102073a8: mov x24, x3 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 3.93 : ffff8000102073ac: mrs x0, sp_el0 : __range_ok(): : * Asynchronous I/O running in a kernel thread does not have the : * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag : * the user address before checking. : */ : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && : (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) 0.00 : ffff8000102073b0: ldr w4, [x0, #44] : unsigned long ret, limit = current_thread_info()->addr_limit; 1.70 : ffff8000102073b4: ldr x3, [x0, #8] : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 0.00 : ffff8000102073b8: tbz w4, #21, ffff80001020741c : sign_extend64(): 0.00 : ffff8000102073bc: sbfx x0, x20, #0, #56 : __range_ok(): : addr = untagged_addr(addr); 0.00 : ffff8000102073c0: and x0, x0, x20 : : __chk_user_ptr(addr); : asm volatile( 0.29 : ffff8000102073c4: adds x0, x0, x2 0.00 : ffff8000102073c8: csel x3, xzr, x3, hi // hi = pmore 0.00 : ffff8000102073cc: csinv x0, x0, xzr, cc // cc = lo, ul, last 0.00 : ffff8000102073d0: sbcs xzr, x0, x3 0.00 : ffff8000102073d4: cset x0, ls // ls = plast : internal_get_user_pages_fast(): : if (unlikely(!access_ok((void __user *)start, len))) 0.00 : ffff8000102073d8: cbz x0, ffff8000102074ec : arch_local_irq_disable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff8000102073dc: mov x0, #0x60 // #96 4.58 : ffff8000102073e0: msr daifset, #0x2 : internal_get_user_pages_fast(): : gup_pgd_range(addr, end, gup_flags, pages, &nr); 10.36 : ffff8000102073e4: mov x0, x20 0.00 : ffff8000102073e8: add x4, x29, #0x44 0.00 : ffff8000102073ec: mov x3, x24 0.00 : ffff8000102073f0: mov w2, w21 0.57 : ffff8000102073f4: bl ffff800010205408 : arch_local_irq_enable(): : asm volatile(ALTERNATIVE( 0.00 : ffff8000102073f8: mov x0, #0xe0 // #224 0.26 : ffff8000102073fc: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 11.47 : ffff800010207400: nop : internal_get_user_pages_fast(): : ret = nr; 0.00 : ffff800010207404: ldr w22, [x29, #68] : if (nr < nr_pages) { 0.00 : ffff800010207408: cmp w23, w22 0.00 : ffff80001020740c: b.gt ffff800010207444 9.73 : ffff800010207410: ldr x20, [x29, #24] 0.00 : ffff800010207414: ldp x23, x24, [x29, #48] 0.00 : ffff800010207418: b ffff80001020737c : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001020741c: ldr x0, [x0] : __range_ok(): : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 0.00 : ffff800010207420: tst w0, #0x4000000 0.00 : ffff800010207424: b.ne ffff8000102073bc // b.any : internal_get_user_pages_fast(): : if (unlikely(!access_ok((void __user *)start, len))) 0.00 : ffff800010207428: mov x0, x20 0.00 : ffff80001020742c: b ffff8000102073c4 : arch_local_irq_enable(): : pmr_sync(); 1.43 : ffff800010207430: dsb sy 48.21 : ffff800010207434: b ffff800010207404 : get_user_pages_fast(): : if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) 0.00 : ffff800010207438: brk #0x800 : return -EINVAL; 0.00 : ffff80001020743c: mov w22, #0xffffffea // #-22 0.00 : ffff800010207440: b ffff80001020737c : internal_get_user_pages_fast(): : start += nr << PAGE_SHIFT; 0.00 : ffff800010207444: lsl w0, w22, #12 : ret = __gup_longterm_unlocked(start, nr_pages - nr, 0.00 : ffff800010207448: sub w23, w23, w22 : pages += nr; 0.00 : ffff80001020744c: add x22, x24, w22, sxtw #3 : start += nr << PAGE_SHIFT; 0.00 : ffff800010207450: add x20, x20, w0, sxtw 0.00 : ffff800010207454: sxtw x23, w23 : __gup_longterm_unlocked(): : if (gup_flags & FOLL_LONGTERM) { 0.00 : ffff800010207458: tbnz w21, #16, ffff8000102074a8 : ret = get_user_pages_unlocked(start, nr_pages, 0.00 : ffff80001020745c: mov x2, x22 0.00 : ffff800010207460: mov w3, w21 0.00 : ffff800010207464: mov x1, x23 0.00 : ffff800010207468: mov x0, x20 0.00 : ffff80001020746c: bl ffff800010206c10 0.00 : ffff800010207470: mov w22, w0 : internal_get_user_pages_fast(): : if (nr > 0) { 0.00 : ffff800010207474: ldr w0, [x29, #68] 0.00 : ffff800010207478: cmp w0, #0x0 0.00 : ffff80001020747c: b.le ffff800010207410 : ret += nr; 0.00 : ffff800010207480: add w1, w22, w0 0.00 : ffff800010207484: cmp w22, #0x0 0.00 : ffff800010207488: csel w22, w1, w0, ge // ge = tcont 0.00 : ffff80001020748c: ldr x20, [x29, #24] 0.00 : ffff800010207490: ldp x23, x24, [x29, #48] 0.00 : ffff800010207494: b ffff80001020737c : if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | 0.00 : ffff800010207498: brk #0x800 : return -EINVAL; 0.00 : ffff80001020749c: mov w22, #0xffffffea // #-22 0.00 : ffff8000102074a0: ldr x23, [x29, #48] 0.00 : ffff8000102074a4: b ffff80001020737c : get_current(): 0.00 : ffff8000102074a8: mrs x24, sp_el0 : __gup_longterm_unlocked(): : down_read(¤t->mm->mmap_sem); 0.00 : ffff8000102074ac: ldr x0, [x24, #952] 0.00 : ffff8000102074b0: add x0, x0, #0x68 0.00 : ffff8000102074b4: bl ffff800010cd62b8 : ret = __gup_longterm_locked(current, current->mm, 0.00 : ffff8000102074b8: ldr x1, [x24, #952] 0.00 : ffff8000102074bc: mov x4, x22 0.00 : ffff8000102074c0: mov w6, w21 0.00 : ffff8000102074c4: mov x3, x23 0.00 : ffff8000102074c8: mov x2, x20 0.00 : ffff8000102074cc: mov x5, #0x0 // #0 0.00 : ffff8000102074d0: mov x0, x24 0.00 : ffff8000102074d4: bl ffff800010206e48 <__gup_longterm_locked> 0.00 : ffff8000102074d8: mov w22, w0 : up_read(¤t->mm->mmap_sem); 0.00 : ffff8000102074dc: ldr x0, [x24, #952] 0.00 : ffff8000102074e0: add x0, x0, #0x68 0.00 : ffff8000102074e4: bl ffff800010138db0 0.00 : ffff8000102074e8: b ffff800010207474 : internal_get_user_pages_fast(): : return -EFAULT; 0.00 : ffff8000102074ec: mov w22, #0xfffffff2 // #-14 0.00 : ffff8000102074f0: ldr x20, [x29, #24] 0.00 : ffff8000102074f4: ldp x23, x24, [x29, #48] 0.00 : ffff8000102074f8: b ffff80001020737c 0.00 : ffff8000102074fc: str x20, [x29, #24] 0.00 : ffff800010207500: stp x23, x24, [x29, #48] : get_user_pages_fast(): : } 0.00 : ffff800010207504: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (190 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101299b8 : : pick_next_task_rt(): : : return rt_task_of(rt_se); : } : : static struct task_struct *pick_next_task_rt(struct rq *rq) : { 23.33 : ffff8000101299b8: stp x29, x30, [sp, #-48]! 0.00 : ffff8000101299bc: mov x29, sp 8.07 : ffff8000101299c0: str x21, [sp, #32] : struct task_struct *p; : : if (!sched_rt_runnable(rq)) 3.23 : ffff8000101299c4: ldr w1, [x0, #2184] 0.00 : ffff8000101299c8: cmp w1, #0x0 0.00 : ffff8000101299cc: b.le ffff800010129af8 0.00 : ffff8000101299d0: stp x19, x20, [x29, #16] : sched_find_first_bit(): : * one of the 100 bits is cleared. : */ : static inline int sched_find_first_bit(const unsigned long *b) : { : #if BITS_PER_LONG == 64 : if (b[0]) 2.03 : ffff8000101299d4: ldr x1, [x0, #512] 0.00 : ffff8000101299d8: cbnz x1, ffff800010129acc : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 0.00 : ffff8000101299dc: ldr x1, [x0, #520] 0.00 : ffff8000101299e0: rbit x1, x1 0.00 : ffff8000101299e4: clz x1, x1 : sched_find_first_bit(): : return __ffs(b[0]); : return __ffs(b[1]) + 64; 0.00 : ffff8000101299e8: add w1, w1, #0x40 : pick_next_rt_entity(): : BUG_ON(idx >= MAX_RT_PRIO); 0.00 : ffff8000101299ec: cmp w1, #0x63 0.00 : ffff8000101299f0: b.hi ffff800010129b0c // b.pmore : next = list_entry(queue->next, struct sched_rt_entity, run_list); 0.54 : ffff8000101299f4: add x1, x0, w1, sxtw #4 0.00 : ffff8000101299f8: ldr x2, [x1, #528] : _pick_next_task_rt(): : BUG_ON(!rt_se); 0.00 : ffff8000101299fc: cbz x2, ffff800010129b10 : dequeue_pushable_task(): : plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 16.76 : ffff800010129a00: add x20, x0, #0x878 0.00 : ffff800010129a04: mov x19, x0 : set_next_task_rt(): : p->se.exec_start = rq_clock_task(rq); 3.74 : ffff800010129a08: ldr x0, [x0, #2432] : rt_task_of(): : return container_of(rt_se, struct task_struct, rt); 0.00 : ffff800010129a0c: sub x21, x2, #0x1c0 : set_next_task_rt(): : p->se.exec_start = rq_clock_task(rq); 0.54 : ffff800010129a10: stur x0, [x2, #-184] : dequeue_pushable_task(): : plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 0.00 : ffff800010129a14: mov x1, x20 0.00 : ffff800010129a18: add x0, x2, #0x1b8 0.00 : ffff800010129a1c: bl ffff800010cc2748 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010129a20: ldr x0, [x19, #2168] : dequeue_pushable_task(): : if (has_pushable_tasks(rq)) { 0.00 : ffff800010129a24: cmp x20, x0 0.00 : ffff800010129a28: b.eq ffff800010129aec // b.none : rq->rt.highest_prio.next = p->prio; 0.00 : ffff800010129a2c: ldr x0, [x19, #2168] 0.00 : ffff800010129a30: sub x0, x0, #0x390 0.00 : ffff800010129a34: ldr w0, [x0, #108] 0.00 : ffff800010129a38: str w0, [x19, #2140] : set_next_task_rt(): : if (rq->curr->sched_class != &rt_sched_class) 0.00 : ffff800010129a3c: ldr x1, [x19, #2352] 0.00 : ffff800010129a40: adrp x0, ffff800010d06000 0.00 : ffff800010129a44: add x0, x0, #0x1f8 7.39 : ffff800010129a48: ldr x1, [x1, #128] 0.00 : ffff800010129a4c: cmp x1, x0 0.00 : ffff800010129a50: b.eq ffff800010129a6c // b.none : rq_clock_pelt(): : static inline u64 rq_clock_pelt(struct rq *rq) : { : lockdep_assert_held(&rq->lock); : assert_clock_updated(rq); : : return rq->clock_pelt - rq->lost_idle_time; 10.57 : ffff800010129a54: ldr x3, [x19, #2440] : set_next_task_rt(): : update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0); 0.00 : ffff800010129a58: mov w2, #0x0 // #0 : rq_clock_pelt(): 1.03 : ffff800010129a5c: ldr x0, [x19, #2448] : set_next_task_rt(): 0.00 : ffff800010129a60: mov x1, x19 0.00 : ffff800010129a64: sub x0, x3, x0 0.00 : ffff800010129a68: bl ffff800010134c40 : __read_once_size(): 0.54 : ffff800010129a6c: ldr x0, [x19, #2168] : rt_queue_push_tasks(): : if (!has_pushable_tasks(rq)) 0.00 : ffff800010129a70: cmp x20, x0 0.00 : ffff800010129a74: b.eq ffff800010129ad8 // b.none : queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks); 0.00 : ffff800010129a78: ldrsw x2, [x19, #2568] 0.00 : ffff800010129a7c: adrp x1, ffff800011909000 0.00 : ffff800010129a80: add x1, x1, #0x928 0.00 : ffff800010129a84: adrp x0, ffff80001151f000 0.00 : ffff800010129a88: add x0, x0, #0x98 0.00 : ffff800010129a8c: add x0, x0, #0x30 0.00 : ffff800010129a90: ldr x1, [x1, x2, lsl #3] 0.00 : ffff800010129a94: add x3, x0, x1 : queue_balance_callback(): : struct callback_head *head, : void (*func)(struct rq *rq)) : { : lockdep_assert_held(&rq->lock); : : if (unlikely(head->next)) 0.00 : ffff800010129a98: ldr x2, [x0, x1] 0.00 : ffff800010129a9c: cbnz x2, ffff800010129ad8 : return; : : head->func = (void (*)(struct callback_head *))func; 0.00 : ffff800010129aa0: adrp x2, ffff800010129000 0.00 : ffff800010129aa4: add x2, x2, #0xe90 0.00 : ffff800010129aa8: str x2, [x3, #8] : head->next = rq->balance_callback; 0.00 : ffff800010129aac: ldr x2, [x19, #2496] 0.00 : ffff800010129ab0: str x2, [x0, x1] : pick_next_task_rt(): : return NULL; : : p = _pick_next_task_rt(rq); : set_next_task_rt(rq, p, true); : return p; : } 0.00 : ffff800010129ab4: mov x0, x21 : queue_balance_callback(): : rq->balance_callback = head; 0.00 : ffff800010129ab8: str x3, [x19, #2496] 0.00 : ffff800010129abc: ldp x19, x20, [x29, #16] : pick_next_task_rt(): 0.00 : ffff800010129ac0: ldr x21, [sp, #32] 0.00 : ffff800010129ac4: ldp x29, x30, [sp], #48 0.00 : ffff800010129ac8: ret : __ffs(): 3.31 : ffff800010129acc: rbit x1, x1 0.00 : ffff800010129ad0: clz x1, x1 0.00 : ffff800010129ad4: b ffff8000101299f4 : pick_next_task_rt(): 0.00 : ffff800010129ad8: mov x0, x21 2.11 : ffff800010129adc: ldr x21, [sp, #32] 3.71 : ffff800010129ae0: ldp x19, x20, [x29, #16] 2.18 : ffff800010129ae4: ldp x29, x30, [sp], #48 0.00 : ffff800010129ae8: ret : dequeue_pushable_task(): : rq->rt.highest_prio.next = MAX_RT_PRIO; 0.00 : ffff800010129aec: mov w0, #0x64 // #100 2.00 : ffff800010129af0: str w0, [x19, #2140] 0.00 : ffff800010129af4: b ffff800010129a3c : pick_next_task_rt(): : return NULL; 1.07 : ffff800010129af8: mov x21, #0x0 // #0 : } 0.00 : ffff800010129afc: mov x0, x21 2.09 : ffff800010129b00: ldr x21, [sp, #32] 5.77 : ffff800010129b04: ldp x29, x30, [sp], #48 0.00 : ffff800010129b08: ret : pick_next_rt_entity(): : BUG_ON(idx >= MAX_RT_PRIO); 0.00 : ffff800010129b0c: brk #0x800 : _pick_next_task_rt(): : BUG_ON(!rt_se); 0.00 : ffff800010129b10: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (357 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d8fa8 <__get_reqs_available>: : __get_reqs_available(): : : local_irq_restore(flags); : } : : static bool __get_reqs_available(struct kioctx *ctx) : { 0.00 : ffff8000102d8fa8: mov x6, x0 : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.28 : ffff8000102d8fac: mrs x8, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000102d8fb0: and w0, w8, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff8000102d8fb4: cbnz w0, ffff8000102d8fc0 <__get_reqs_available+0x18> : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff8000102d8fb8: mov x0, #0x60 // #96 0.00 : ffff8000102d8fbc: msr daifset, #0x2 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 9.45 : ffff8000102d8fc0: mrs x9, tpidr_el1 : __get_reqs_available(): : struct kioctx_cpu *kcpu; : bool ret = false; : unsigned long flags; : : local_irq_save(flags); : kcpu = this_cpu_ptr(ctx->cpu); 0.28 : ffff8000102d8fc4: ldr x5, [x6, #128] : if (!kcpu->reqs_available) { 3.64 : ffff8000102d8fc8: ldr w0, [x5, x9] 0.00 : ffff8000102d8fcc: cbnz w0, ffff8000102d903c <__get_reqs_available+0x94> : atomic_cmpxchg(): : #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) : static inline int : atomic_cmpxchg(atomic_t *v, int old, int new) : { : kasan_check_write(v, sizeof(*v)); : return arch_atomic_cmpxchg(v, old, new); 2.27 : ffff8000102d8fd0: add x3, x6, #0x100 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.28 : ffff8000102d8fd4: ldr w4, [x6, #256] 0.00 : ffff8000102d8fd8: b ffff8000102d9004 <__get_reqs_available+0x5c> : __get_reqs_available(): : if (avail < ctx->req_batch) : goto out; : : old = avail; : avail = atomic_cmpxchg(&ctx->reqs_available, : avail, avail - ctx->req_batch); 0.28 : ffff8000102d8fdc: sub w2, w4, w2 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102d8fe0: b ffff8000102d901c <__get_reqs_available+0x74> 0.28 : ffff8000102d8fe4: b ffff8000102d901c <__get_reqs_available+0x74> : __lse__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 0.00 : ffff8000102d8fe8: mov x0, x3 0.00 : ffff8000102d8fec: mov w7, w1 0.00 : ffff8000102d8ff0: casal w7, w2, [x3] 8.13 : ffff8000102d8ff4: mov w0, w7 : __get_reqs_available(): : } while (avail != old); 0.00 : ffff8000102d8ff8: cmp w4, w0 0.00 : ffff8000102d8ffc: mov w4, w0 0.00 : ffff8000102d9000: b.eq ffff8000102d9030 <__get_reqs_available+0x88> // b.none : if (avail < ctx->req_batch) 1.43 : ffff8000102d9004: ldr w2, [x6, #136] : atomic_cmpxchg(): 0.00 : ffff8000102d9008: sxtw x1, w4 : __get_reqs_available(): 0.00 : ffff8000102d900c: cmp w2, w4 0.00 : ffff8000102d9010: b.ls ffff8000102d8fdc <__get_reqs_available+0x34> // b.plast : bool ret = false; 0.00 : ffff8000102d9014: mov w0, #0x0 // #0 0.00 : ffff8000102d9018: b ffff8000102d9048 <__get_reqs_available+0xa0> : __ll_sc__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 0.00 : ffff8000102d901c: and x1, x1, #0xffffffff 0.00 : ffff8000102d9020: b ffff8000102dd868 <__arm64_compat_sys_io_pgetevents_time64+0x1f8> : __get_reqs_available(): : } while (avail != old); 0.00 : ffff8000102d9024: cmp w4, w0 0.00 : ffff8000102d9028: mov w4, w0 0.00 : ffff8000102d902c: b.ne ffff8000102d9004 <__get_reqs_available+0x5c> // b.any : : kcpu->reqs_available += ctx->req_batch; 0.00 : ffff8000102d9030: ldr w0, [x5, x9] 0.28 : ffff8000102d9034: ldr w1, [x6, #136] 0.00 : ffff8000102d9038: add w0, w0, w1 : } : : ret = true; : kcpu->reqs_available--; 2.84 : ffff8000102d903c: sub w1, w0, #0x1 : ret = true; 0.00 : ffff8000102d9040: mov w0, #0x1 // #1 : kcpu->reqs_available--; 0.00 : ffff8000102d9044: str w1, [x5, x9] : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.84 : ffff8000102d9048: msr daif, x8 : arch_static_branch(): : asm_volatile_goto( 12.36 : ffff8000102d904c: nop : __get_reqs_available(): : out: : local_irq_restore(flags); : return ret; : } 0.00 : ffff8000102d9050: ret : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 1.13 : ffff8000102d9054: dsb sy : __get_reqs_available(): 56.20 : ffff8000102d9058: ret Percent | Source code & Disassembly of vmlinux for cycles (133 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016b020 <__hrtimer_get_next_event>: : __hrtimer_get_next_event(): : * - HRTIMER_ACTIVE_SOFT, or : * - HRTIMER_ACTIVE_HARD. : */ : static ktime_t : __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask) : { 0.00 : ffff80001016b020: stp x29, x30, [sp, #-32]! : unsigned int active; : struct hrtimer *next_timer = NULL; : ktime_t expires_next = KTIME_MAX; 0.00 : ffff80001016b024: mov x3, #0x7fffffffffffffff // #9223372036854775807 : { 0.00 : ffff80001016b028: mov x29, sp 22.58 : ffff80001016b02c: stp x19, x20, [sp, #16] 0.00 : ffff80001016b030: mov w20, w1 0.00 : ffff80001016b034: mov x19, x0 : : if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) { 8.23 : ffff80001016b038: ldrb w1, [x0, #16] 0.00 : ffff80001016b03c: tbnz w1, #3, ffff80001016b080 <__hrtimer_get_next_event+0x60> : struct hrtimer *next_timer = NULL; 2.99 : ffff80001016b040: mov x1, #0x0 // #0 : if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) { 0.00 : ffff80001016b044: tst w20, #0xf0 0.00 : ffff80001016b048: b.ne ffff80001016b088 <__hrtimer_get_next_event+0x68> // b.any : active, KTIME_MAX); : : next_timer = cpu_base->softirq_next_timer; : } : : if (active_mask & HRTIMER_ACTIVE_HARD) { 6.76 : ffff80001016b04c: tst x20, #0xf 0.00 : ffff80001016b050: b.eq ffff80001016b070 <__hrtimer_get_next_event+0x50> // b.none : active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; 0.00 : ffff80001016b054: ldr w2, [x19, #8] : cpu_base->next_timer = next_timer; : expires_next = __hrtimer_next_event_base(cpu_base, NULL, active, 0.00 : ffff80001016b058: mov x0, x19 : cpu_base->next_timer = next_timer; 14.79 : ffff80001016b05c: str x1, [x19, #40] : expires_next = __hrtimer_next_event_base(cpu_base, NULL, active, 0.00 : ffff80001016b060: mov x1, #0x0 // #0 0.00 : ffff80001016b064: and w2, w2, #0xf 0.00 : ffff80001016b068: bl ffff80001016af18 <__hrtimer_next_event_base> 0.79 : ffff80001016b06c: mov x3, x0 : expires_next); : } : : return expires_next; : } 0.00 : ffff80001016b070: mov x0, x3 0.00 : ffff80001016b074: ldp x19, x20, [sp, #16] 4.48 : ffff80001016b078: ldp x29, x30, [sp], #32 0.00 : ffff80001016b07c: ret : struct hrtimer *next_timer = NULL; 0.00 : ffff80001016b080: mov x1, #0x0 // #0 0.00 : ffff80001016b084: b ffff80001016b04c <__hrtimer_get_next_event+0x2c> : active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; 7.52 : ffff80001016b088: ldr w2, [x0, #8] : cpu_base->softirq_next_timer = NULL; 8.97 : ffff80001016b08c: str xzr, [x0, #56] : expires_next = __hrtimer_next_event_base(cpu_base, NULL, 0.00 : ffff80001016b090: and w2, w2, #0xf0 0.00 : ffff80001016b094: bl ffff80001016af18 <__hrtimer_next_event_base> 19.26 : ffff80001016b098: mov x3, x0 : next_timer = cpu_base->softirq_next_timer; 3.64 : ffff80001016b09c: ldr x1, [x19, #56] 0.00 : ffff80001016b0a0: b ffff80001016b04c <__hrtimer_get_next_event+0x2c> Percent | Source code & Disassembly of vmlinux for cycles (129 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016b130 <__remove_hrtimer>: : __remove_hrtimer(): : * anyway (e.g. timer interrupt) : */ : static void __remove_hrtimer(struct hrtimer *timer, : struct hrtimer_clock_base *base, : u8 newstate, int reprogram) : { 2.35 : ffff80001016b130: stp x29, x30, [sp, #-48]! 0.00 : ffff80001016b134: and w2, w2, #0xff 0.00 : ffff80001016b138: mov x29, sp 3.92 : ffff80001016b13c: str x22, [sp, #40] : struct hrtimer_cpu_base *cpu_base = base->cpu_base; : u8 state = timer->state; 2.14 : ffff80001016b140: ldrb w4, [x0, #56] : struct hrtimer_cpu_base *cpu_base = base->cpu_base; 3.89 : ffff80001016b144: ldr x22, [x1] : __write_once_size(): : } : : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 1.57 : ffff80001016b148: strb w2, [x0, #56] : __remove_hrtimer(): : : /* Pairs with the lockless read in hrtimer_is_queued() */ : WRITE_ONCE(timer->state, newstate); : if (!(state & HRTIMER_STATE_ENQUEUED)) 0.00 : ffff80001016b14c: tbnz w4, #0, ffff80001016b15c <__remove_hrtimer+0x2c> : * an superflous call to hrtimer_force_reprogram() on the : * remote cpu later on if the same timer gets enqueued again. : */ : if (reprogram && timer == cpu_base->next_timer) : hrtimer_force_reprogram(cpu_base, 1); : } 0.00 : ffff80001016b150: ldr x22, [sp, #40] 0.00 : ffff80001016b154: ldp x29, x30, [sp], #48 0.00 : ffff80001016b158: ret 0.00 : ffff80001016b15c: stp x19, x20, [x29, #16] 0.00 : ffff80001016b160: mov x20, x1 0.00 : ffff80001016b164: str x21, [x29, #32] : if (!timerqueue_del(&base->active, &timer->node)) 0.00 : ffff80001016b168: mov x1, x0 0.00 : ffff80001016b16c: mov x19, x0 0.00 : ffff80001016b170: mov w21, w3 3.11 : ffff80001016b174: add x0, x20, #0x20 0.00 : ffff80001016b178: bl ffff800010cc8b58 3.88 : ffff80001016b17c: tst w0, #0xff 0.00 : ffff80001016b180: b.ne ffff80001016b19c <__remove_hrtimer+0x6c> // b.any : cpu_base->active_bases &= ~(1 << base->index); 0.00 : ffff80001016b184: ldr w2, [x20, #8] 0.00 : ffff80001016b188: mov w0, #0x1 // #1 0.00 : ffff80001016b18c: ldr w1, [x22, #8] 0.00 : ffff80001016b190: lsl w0, w0, w2 0.00 : ffff80001016b194: bic w0, w1, w0 0.00 : ffff80001016b198: str w0, [x22, #8] : if (reprogram && timer == cpu_base->next_timer) 7.86 : ffff80001016b19c: cbz w21, ffff80001016b1ac <__remove_hrtimer+0x7c> 10.18 : ffff80001016b1a0: ldr x0, [x22, #40] 0.00 : ffff80001016b1a4: cmp x0, x19 0.00 : ffff80001016b1a8: b.eq ffff80001016b1c0 <__remove_hrtimer+0x90> // b.none 0.00 : ffff80001016b1ac: ldp x19, x20, [x29, #16] 0.77 : ffff80001016b1b0: ldr x21, [x29, #32] : } 0.00 : ffff80001016b1b4: ldr x22, [sp, #40] 1.66 : ffff80001016b1b8: ldp x29, x30, [sp], #48 0.00 : ffff80001016b1bc: ret : hrtimer_force_reprogram(cpu_base, 1); 19.50 : ffff80001016b1c0: mov w1, #0x1 // #1 0.00 : ffff80001016b1c4: mov x0, x22 0.00 : ffff80001016b1c8: bl ffff80001016b0a8 6.03 : ffff80001016b1cc: ldr x21, [x29, #32] 33.13 : ffff80001016b1d0: ldp x19, x20, [x29, #16] 0.00 : ffff80001016b1d4: b ffff80001016b150 <__remove_hrtimer+0x20> Percent | Source code & Disassembly of vmlinux for cycles (231 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010085bf8 : : fpsimd_thread_switch(): : (void __user *)instruction_pointer(regs), : current); : } : : void fpsimd_thread_switch(struct task_struct *next) : { 2.61 : ffff800010085bf8: stp x29, x30, [sp, #-32]! 0.00 : ffff800010085bfc: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010085c00: b ffff800010085c68 : arch_static_branch(): : asm_volatile_goto( 3.91 : ffff800010085c04: nop 0.00 : ffff800010085c08: str x19, [x29, #16] 0.00 : ffff800010085c0c: mov x19, x0 : fpsimd_thread_switch(): : bool wrong_task, wrong_cpu; : : if (!system_supports_fpsimd()) : return; : : __get_cpu_fpsimd_context(); 0.00 : ffff800010085c10: bl ffff800010085378 <__get_cpu_fpsimd_context> : : /* Save unsaved fpsimd state, if any: */ : fpsimd_save(); 5.35 : ffff800010085c14: bl ffff800010085588 : * state. For kernel threads, FPSIMD registers are never loaded : * and wrong_task and wrong_cpu will always be true. : */ : wrong_task = __this_cpu_read(fpsimd_last_state.st) != : &next->thread.uw.fpsimd_state; : wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id(); 0.88 : ffff800010085c18: adrp x1, ffff80001151d000 0.00 : ffff800010085c1c: add x1, x1, #0x18 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 3.87 : ffff800010085c20: mrs x2, tpidr_el1 : fpsimd_thread_switch(): : : update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, 0.00 : ffff800010085c24: ldr w3, [x19, #3152] : wrong_task = __this_cpu_read(fpsimd_last_state.st) != 0.89 : ffff800010085c28: adrp x0, ffff80001151d000 : update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, 31.29 : ffff800010085c2c: ldr w4, [x1, x2] : wrong_task = __this_cpu_read(fpsimd_last_state.st) != 0.00 : ffff800010085c30: add x0, x0, #0x60 : update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, 4.52 : ffff800010085c34: ldr x1, [x0, x2] 0.00 : ffff800010085c38: cmp w4, w3 : &next->thread.uw.fpsimd_state; 0.00 : ffff800010085c3c: add x0, x19, #0xa40 : update_ti_thread_flag(): : } : : static inline void update_ti_thread_flag(struct thread_info *ti, int flag, : bool value) : { : if (value) 0.00 : ffff800010085c40: ccmp x1, x0, #0x0, eq // eq = none 1.74 : ffff800010085c44: b.ne ffff800010085c80 // b.any : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff800010085c48: b ffff800010085c94 0.46 : ffff800010085c4c: b ffff800010085c94 : __lse_atomic64_andnot(): : " " #asm_op " %[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) 0.00 : ffff800010085c50: mov x0, #0x8 // #8 0.00 : ffff800010085c54: stclr x0, [x19] : fpsimd_thread_switch(): : wrong_task || wrong_cpu); : : __put_cpu_fpsimd_context(); 1.87 : ffff800010085c58: bl ffff8000100853a8 <__put_cpu_fpsimd_context> 4.79 : ffff800010085c5c: ldr x19, [x29, #16] : } 1.32 : ffff800010085c60: ldp x29, x30, [sp], #32 0.00 : ffff800010085c64: ret : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010085c68: adrp x1, ffff800011af2000 0.00 : ffff800010085c6c: ldr x1, [x1, #1320] : fpsimd_thread_switch(): : if (!system_supports_fpsimd()) 0.00 : ffff800010085c70: tst w1, #0x10000 0.00 : ffff800010085c74: b.ne ffff800010085c60 // b.any 0.00 : ffff800010085c78: str x19, [x29, #16] 0.00 : ffff800010085c7c: b ffff800010085c0c : arch_static_branch_jump(): 0.83 : ffff800010085c80: b ffff800010085ca0 9.06 : ffff800010085c84: b ffff800010085ca0 : __lse_atomic64_or(): : ATOMIC64_OP(or, stset) 0.00 : ffff800010085c88: mov x0, #0x8 // #8 0.00 : ffff800010085c8c: stset x0, [x19] 26.61 : ffff800010085c90: b ffff800010085c58 : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff800010085c94: mov x0, #0x8 // #8 0.00 : ffff800010085c98: b ffff80001008686c <__efi_fpsimd_end+0x15c> 0.00 : ffff800010085c9c: b ffff800010085c58 : __ll_sc_atomic64_or(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010085ca0: b ffff800010086884 <__efi_fpsimd_end+0x174> 0.00 : ffff800010085ca4: b ffff800010085c58 Percent | Source code & Disassembly of vmlinux for cycles (208 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010134550 : : pick_next_task_stop(): : { : stop->se.exec_start = rq_clock_task(rq); : } : : static struct task_struct *pick_next_task_stop(struct rq *rq) : { 13.33 : ffff800010134550: mov x1, x0 : if (!sched_stop_runnable(rq)) 4.71 : ffff800010134554: ldr x0, [x0, #2368] : sched_stop_runnable(): : extern const struct sched_class fair_sched_class; : extern const struct sched_class idle_sched_class; : : static inline bool sched_stop_runnable(struct rq *rq) : { : return rq->stop && task_on_rq_queued(rq->stop); 0.00 : ffff800010134558: cbz x0, ffff80001013456c 29.47 : ffff80001013455c: ldr w2, [x0, #104] 32.13 : ffff800010134560: cmp w2, #0x1 0.00 : ffff800010134564: b.eq ffff800010134570 // b.none : pick_next_task_stop(): : return NULL; 20.36 : ffff800010134568: mov x0, #0x0 // #0 : : set_next_task_stop(rq, rq->stop, true); : return rq->stop; : } 0.00 : ffff80001013456c: ret : set_next_task_stop(): : stop->se.exec_start = rq_clock_task(rq); 0.00 : ffff800010134570: ldr x2, [x1, #2432] 0.00 : ffff800010134574: str x2, [x0, #264] : pick_next_task_stop(): : return rq->stop; 0.00 : ffff800010134578: ldr x0, [x1, #2368] : } 0.00 : ffff80001013457c: ret Percent | Source code & Disassembly of vmlinux for cycles (108 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016a9c8 : : run_local_timers(): : : /* : * Called by the local, per-CPU timer interrupt on SMP. : */ : void run_local_timers(void) : { 0.00 : ffff80001016a9c8: stp x29, x30, [sp, #-32]! : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); 0.00 : ffff80001016a9cc: adrp x0, ffff800011523000 0.00 : ffff80001016a9d0: add x0, x0, #0x480 : { 0.00 : ffff80001016a9d4: mov x29, sp 5.61 : ffff80001016a9d8: str x19, [sp, #16] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001016a9dc: mrs x19, tpidr_el1 : run_local_timers(): : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); 0.00 : ffff80001016a9e0: add x19, x0, x19 : : hrtimer_run_queues(); 0.00 : ffff80001016a9e4: bl ffff80001016bf00 : /* Raise the softirq only if required. */ : if (time_before(jiffies, base->clk)) { 0.00 : ffff80001016a9e8: adrp x1, ffff800011907000 0.00 : ffff80001016a9ec: ldr x0, [x1, #2432] 85.84 : ffff80001016a9f0: ldr x2, [x19, #16] 0.00 : ffff80001016a9f4: sub x0, x0, x2 0.00 : ffff80001016a9f8: tbz x0, #63, ffff80001016aa0c : if (!IS_ENABLED(CONFIG_NO_HZ_COMMON)) : return; : /* CPU is awake, so check the deferrable base. */ : base++; : if (time_before(jiffies, base->clk)) 0.00 : ffff80001016a9fc: ldr x0, [x1, #2432] 3.86 : ffff80001016aa00: ldr x1, [x19, #4752] 0.00 : ffff80001016aa04: sub x0, x0, x1 0.00 : ffff80001016aa08: tbnz x0, #63, ffff80001016aa14 : return; : } : raise_softirq(TIMER_SOFTIRQ); 2.87 : ffff80001016aa0c: mov w0, #0x1 // #1 0.00 : ffff80001016aa10: bl ffff8000100ecde0 : } 0.94 : ffff80001016aa14: ldr x19, [sp, #16] 0.88 : ffff80001016aa18: ldp x29, x30, [sp], #32 0.00 : ffff80001016aa1c: ret Percent | Source code & Disassembly of vmlinux for cycles (204 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010114248 : : finish_task_switch(): : * past. prev == current is still correct but we need to recalculate this_rq : * because prev may have moved to another CPU. : */ : static struct rq *finish_task_switch(struct task_struct *prev) : __releases(rq->lock) : { 4.40 : ffff800010114248: stp x29, x30, [sp, #-64]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.93 : ffff80001011424c: mrs x1, sp_el0 : finish_task_switch(): 0.00 : ffff800010114250: mov x29, sp 0.00 : ffff800010114254: stp x19, x20, [sp, #16] : struct rq *rq = this_rq(); 0.00 : ffff800010114258: adrp x19, ffff800011528000 : { 7.87 : ffff80001011425c: stp x21, x22, [sp, #32] : struct rq *rq = this_rq(); 0.00 : ffff800010114260: add x19, x19, #0xe80 : { 2.41 : ffff800010114264: stp x23, x24, [sp, #48] 0.00 : ffff800010114268: mov x22, x0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.97 : ffff80001011426c: mrs x3, tpidr_el1 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010114270: ldr w2, [x1, #16] 0.00 : ffff800010114274: mov x24, x30 : finish_task_switch(): : struct rq *rq = this_rq(); 0.00 : ffff800010114278: add x19, x19, x3 : * __schedule() : * raw_spin_lock_irq(&rq->lock) // 2 : * : * Also, see FORK_PREEMPT_COUNT. : */ : if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 0.00 : ffff80001011427c: cmp w2, #0x2 : struct mm_struct *mm = rq->prev_mm; 51.15 : ffff800010114280: ldr x20, [x19, #2384] : if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 0.00 : ffff800010114284: b.ne ffff800010114400 // b.any : "corrupted preempt_count: %s/%d/0x%x\n", : current->comm, current->pid, preempt_count())) : preempt_count_set(FORK_PREEMPT_COUNT); : : rq->prev_mm = NULL; 14.70 : ffff800010114288: str xzr, [x19, #2384] : get_current(): 0.95 : ffff80001011428c: mrs x21, sp_el0 : finish_task_switch(): : * We must observe prev->state before clearing prev->on_cpu (in : * finish_task), otherwise a concurrent wakeup can get prev : * running on another CPU and we could rave with its RUNNING -> DEAD : * transition, resulting in a double drop. : */ : prev_state = prev->state; 0.00 : ffff800010114290: ldr x23, [x22, #24] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010114294: nop 0.00 : ffff800010114298: nop : finish_task(): : smp_store_release(&prev->on_cpu, 0); 4.35 : ffff80001011429c: mov w0, #0x0 // #0 0.00 : ffff8000101142a0: add x1, x22, #0x40 0.00 : ffff8000101142a4: stlr w0, [x1] : finish_lock_switch(): : raw_spin_unlock_irq(&rq->lock); 0.00 : ffff8000101142a8: mov x0, x19 0.00 : ffff8000101142ac: bl ffff800010cd7c70 <_raw_spin_unlock_irq> : get_current(): 0.44 : ffff8000101142b0: mrs x0, sp_el0 : arch_static_branch(): 0.00 : ffff8000101142b4: nop : finish_task_switch(): : * : * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly : * provided by mmdrop(), : * - a sync_core for SYNC_CORE. : */ : if (mm) { 0.00 : ffff8000101142b8: cbz x20, ffff8000101142ec : get_current(): 0.00 : ffff8000101142bc: mrs x0, sp_el0 : membarrier_mm_sync_core_before_usermode(): : #include : #endif : : static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) : { : if (current->mm != mm) 0.00 : ffff8000101142c0: ldr x0, [x0, #952] 0.00 : ffff8000101142c4: cmp x20, x0 0.00 : ffff8000101142c8: b.eq ffff8000101143b4 // b.none : mmdrop(): : if (unlikely(atomic_dec_and_test(&mm->mm_count))) 0.00 : ffff8000101142cc: add x1, x20, #0x50 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101142d0: b ffff80001011430c 0.00 : ffff8000101142d4: b ffff80001011430c : __lse_atomic_sub_return(): : } : : ATOMIC_OP_SUB_RETURN(_relaxed, ) : ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC_OP_SUB_RETURN(_release, l, "memory") : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000101142d8: mov w0, #0x1 // #1 0.00 : ffff8000101142dc: neg w0, w0 0.00 : ffff8000101142e0: ldaddal w0, w2, [x1] 7.34 : ffff8000101142e4: add w0, w0, w2 : mmdrop(): 0.00 : ffff8000101142e8: cbz w0, ffff80001011441c : finish_task_switch(): : membarrier_mm_sync_core_before_usermode(mm); : mmdrop(mm); : } : if (unlikely(prev_state == TASK_DEAD)) { 0.00 : ffff8000101142ec: cmp x23, #0x80 0.00 : ffff8000101142f0: b.eq ffff8000101143c4 // b.none : put_task_struct_rcu_user(prev); : } : : tick_nohz_task_switch(); : return rq; : } 1.42 : ffff8000101142f4: mov x0, x19 0.00 : ffff8000101142f8: ldp x19, x20, [sp, #16] 0.00 : ffff8000101142fc: ldp x21, x22, [sp, #32] 0.50 : ffff800010114300: ldp x23, x24, [sp, #48] 0.00 : ffff800010114304: ldp x29, x30, [sp], #64 0.00 : ffff800010114308: ret : __ll_sc_atomic_sub_return(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff80001011430c: mov w1, #0x1 // #1 0.00 : ffff800010114310: add x3, x20, #0x50 0.00 : ffff800010114314: b ffff80001011aa18 0.00 : ffff800010114318: b ffff8000101142e8 : perf_event_task_sched_in(): : struct task_struct *task) : { : if (static_branch_unlikely(&perf_sched_events)) : __perf_event_task_sched_in(prev, task); : : if (perf_sw_migrate_enabled() && task->sched_migrated) { 0.00 : ffff80001011431c: ldrb w0, [x21, #1060] 0.00 : ffff800010114320: tbz w0, #2, ffff80001011429c : __my_cpu_offset(): : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff800010114324: mov x5, sp : perf_event_task_sched_in(): : struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]); 0.00 : ffff800010114328: adrp x4, ffff800011526000 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011432c: mrs x0, tpidr_el1 : perf_event_task_sched_in(): 0.00 : ffff800010114330: add x4, x4, #0x150 : perf_fetch_caller_regs(): : perf_arch_fetch_caller_regs(regs, CALLER_ADDR0); 0.00 : ffff800010114334: mov x6, #0x5 // #5 : perf_event_task_sched_in(): : struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]); 0.00 : ffff800010114338: add x4, x4, x0 : : perf_fetch_caller_regs(regs); : ___perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, regs, 0); 0.00 : ffff80001011433c: mov x3, #0x0 // #0 0.00 : ffff800010114340: mov w0, #0x4 // #4 0.00 : ffff800010114344: mov x2, x4 0.00 : ffff800010114348: mov x1, #0x1 // #1 : perf_fetch_caller_regs(): : perf_arch_fetch_caller_regs(regs, CALLER_ADDR0); 0.00 : ffff80001011434c: str x29, [x4, #232] 0.00 : ffff800010114350: stp x5, x24, [x4, #248] 0.00 : ffff800010114354: str x6, [x4, #264] : perf_event_task_sched_in(): : ___perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, regs, 0); 0.00 : ffff800010114358: bl ffff8000101ca150 <___perf_sw_event> : task->sched_migrated = 0; 0.00 : ffff80001011435c: ldrb w0, [x21, #1060] 0.00 : ffff800010114360: and w0, w0, #0xfffffffb 0.00 : ffff800010114364: strb w0, [x21, #1060] 0.00 : ffff800010114368: b ffff80001011429c : __fire_sched_in_preempt_notifiers(): : hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 0.00 : ffff80001011436c: ldr x21, [x0, #720] 0.00 : ffff800010114370: cbz x21, ffff8000101142b8 0.00 : ffff800010114374: adrp x24, ffff80001151d000 : notifier->ops->sched_in(notifier, raw_smp_processor_id()); 0.00 : ffff800010114378: add x24, x24, #0x18 0.00 : ffff80001011437c: ldr x2, [x21, #16] 0.00 : ffff800010114380: mov x0, x24 : __my_cpu_offset(): 0.00 : ffff800010114384: mrs x1, tpidr_el1 : __fire_sched_in_preempt_notifiers(): 0.00 : ffff800010114388: ldr x2, [x2] 0.00 : ffff80001011438c: ldr w1, [x0, x1] 0.00 : ffff800010114390: mov x0, x21 0.00 : ffff800010114394: blr x2 : hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 0.00 : ffff800010114398: ldr x21, [x21] 0.00 : ffff80001011439c: cbnz x21, ffff80001011437c 0.00 : ffff8000101143a0: b ffff8000101142b8 : perf_event_task_sched_in(): : __perf_event_task_sched_in(prev, task); 0.00 : ffff8000101143a4: mov x1, x21 0.00 : ffff8000101143a8: mov x0, x22 0.00 : ffff8000101143ac: bl ffff8000101c41b8 <__perf_event_task_sched_in> 0.00 : ffff8000101143b0: b ffff800010114298 : __read_once_size(): 0.00 : ffff8000101143b4: ldr w0, [x20, #72] : mmdrop(): 0.00 : ffff8000101143b8: add x1, x20, #0x50 : arch_static_branch_jump(): 0.00 : ffff8000101143bc: b ffff80001011430c 2.57 : ffff8000101143c0: b ffff8000101142d4 : finish_task_switch(): : if (prev->sched_class->task_dead) 0.00 : ffff8000101143c4: ldr x0, [x22, #128] 0.00 : ffff8000101143c8: ldr x1, [x0, #144] 0.00 : ffff8000101143cc: cbz x1, ffff8000101143d8 : prev->sched_class->task_dead(prev); 0.00 : ffff8000101143d0: mov x0, x22 0.00 : ffff8000101143d4: blr x1 : put_task_stack(prev); 0.00 : ffff8000101143d8: mov x0, x22 0.00 : ffff8000101143dc: bl ffff8000100e34e0 : put_task_struct_rcu_user(prev); 0.00 : ffff8000101143e0: mov x0, x22 0.00 : ffff8000101143e4: bl ffff8000100e9618 : } 0.00 : ffff8000101143e8: mov x0, x19 0.00 : ffff8000101143ec: ldp x19, x20, [sp, #16] 0.00 : ffff8000101143f0: ldp x21, x22, [sp, #32] 0.00 : ffff8000101143f4: ldp x23, x24, [sp, #48] 0.00 : ffff8000101143f8: ldp x29, x30, [sp], #64 0.00 : ffff8000101143fc: ret : if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 0.00 : ffff800010114400: adrp x4, ffff800011ad8000 0.00 : ffff800010114404: ldrb w0, [x4, #1208] 0.00 : ffff800010114408: cbz w0, ffff800010114428 : get_current(): 0.00 : ffff80001011440c: mrs x0, sp_el0 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010114410: mov w1, #0x2 // #2 0.00 : ffff800010114414: str w1, [x0, #16] 0.00 : ffff800010114418: b ffff800010114288 : mmdrop(): : __mmdrop(mm); 0.00 : ffff80001011441c: mov x0, x20 0.00 : ffff800010114420: bl ffff8000100e2628 <__mmdrop> 0.00 : ffff800010114424: b ffff8000101142ec : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010114428: ldr w3, [x1, #16] : finish_task_switch(): 0.00 : ffff80001011442c: mov w5, #0x1 // #1 0.00 : ffff800010114430: ldr w2, [x1, #1128] 0.00 : ffff800010114434: adrp x0, ffff8000111c3000 0.00 : ffff800010114438: strb w5, [x4, #1208] 0.00 : ffff80001011443c: add x0, x0, #0xfa8 0.00 : ffff800010114440: add x1, x1, #0x620 0.00 : ffff800010114444: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010114448: brk #0x800 0.00 : ffff80001011444c: b ffff80001011440c Percent | Source code & Disassembly of vmlinux for cycles (109 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016b0a8 : : hrtimer_force_reprogram(): : * next event : * Called with interrupts disabled and base->lock held : */ : static void : hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal) : { 11.05 : ffff80001016b0a8: stp x29, x30, [sp, #-32]! 0.00 : ffff80001016b0ac: mov x29, sp 37.60 : ffff80001016b0b0: stp x19, x20, [sp, #16] 0.00 : ffff80001016b0b4: mov x19, x0 0.00 : ffff80001016b0b8: mov w20, w1 : ktime_t expires_next; : : /* : * Find the current next expiration time. : */ : expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL); 0.00 : ffff80001016b0bc: mov w1, #0xff // #255 13.66 : ffff80001016b0c0: bl ffff80001016b020 <__hrtimer_get_next_event> : : if (cpu_base->next_timer && cpu_base->next_timer->is_soft) { 0.00 : ffff80001016b0c4: ldr x1, [x19, #40] 0.00 : ffff80001016b0c8: cbz x1, ffff80001016b0e0 0.00 : ffff80001016b0cc: ldrb w1, [x1, #58] 0.00 : ffff80001016b0d0: cbz w1, ffff80001016b0e0 : /* : * When the softirq is activated, hrtimer has to be : * programmed with the first hard hrtimer because soft : * timer interrupt could occur too late. : */ : if (cpu_base->softirq_activated) 0.00 : ffff80001016b0d4: ldrb w1, [x19, #16] 0.00 : ffff80001016b0d8: tbnz w1, #3, ffff80001016b120 : expires_next = __hrtimer_get_next_event(cpu_base, : HRTIMER_ACTIVE_HARD); : else : cpu_base->softirq_expires_next = expires_next; 0.00 : ffff80001016b0dc: str x0, [x19, #48] : } : : if (skip_equal && expires_next == cpu_base->expires_next) 9.36 : ffff80001016b0e0: cbz w20, ffff80001016b0f0 8.13 : ffff80001016b0e4: ldr x1, [x19, #32] 0.00 : ffff80001016b0e8: cmp x1, x0 0.00 : ffff80001016b0ec: b.eq ffff80001016b100 // b.none : __hrtimer_hres_active(): : cpu_base->hres_active : 0; 0.00 : ffff80001016b0f0: ldrb w1, [x19, #16] : hrtimer_force_reprogram(): : return; : : cpu_base->expires_next = expires_next; 0.00 : ffff80001016b0f4: str x0, [x19, #32] : * the hardware to 5s from now. Any hrtimer_start after that : * will not reprogram the hardware due to hang_detected being : * set. So we'd effectivly block all timers until the T2 event : * fires. : */ : if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected) 0.00 : ffff80001016b0f8: tbz w1, #0, ffff80001016b100 0.00 : ffff80001016b0fc: tbz w1, #2, ffff80001016b10c : return; : : tick_program_event(cpu_base->expires_next, 1); : } 0.00 : ffff80001016b100: ldp x19, x20, [sp, #16] 0.00 : ffff80001016b104: ldp x29, x30, [sp], #32 0.00 : ffff80001016b108: ret : tick_program_event(cpu_base->expires_next, 1); 8.31 : ffff80001016b10c: mov w1, #0x1 // #1 0.00 : ffff80001016b110: bl ffff80001017afa8 : } 0.00 : ffff80001016b114: ldp x19, x20, [sp, #16] 11.90 : ffff80001016b118: ldp x29, x30, [sp], #32 0.00 : ffff80001016b11c: ret : expires_next = __hrtimer_get_next_event(cpu_base, 0.00 : ffff80001016b120: mov w1, #0xf // #15 0.00 : ffff80001016b124: mov x0, x19 0.00 : ffff80001016b128: bl ffff80001016b020 <__hrtimer_get_next_event> 0.00 : ffff80001016b12c: b ffff80001016b0e0 Percent | Source code & Disassembly of vmlinux for cycles (109 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b590 <__tick_nohz_idle_restart_tick>: : __tick_nohz_idle_restart_tick(): : account_idle_ticks(ticks); : #endif : } : : static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now) : { 0.00 : ffff80001017b590: stp x29, x30, [sp, #-48]! : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001017b594: adrp x2, ffff800011b22000 : __tick_nohz_idle_restart_tick(): 0.00 : ffff80001017b598: mov x29, sp 10.05 : ffff80001017b59c: stp x19, x20, [sp, #16] 0.00 : ffff80001017b5a0: mov x19, x0 0.00 : ffff80001017b5a4: str x21, [sp, #32] : tick_do_update_jiffies64(): : if (delta < tick_period) 0.00 : ffff80001017b5a8: adrp x21, ffff800011b22000 : __read_once_size(): 0.00 : ffff80001017b5ac: ldr x0, [x2, #1328] : __tick_nohz_idle_restart_tick(): : { 0.00 : ffff80001017b5b0: mov x20, x1 : tick_do_update_jiffies64(): : delta = ktime_sub(now, READ_ONCE(last_jiffies_update)); 0.00 : ffff80001017b5b4: sub x0, x1, x0 : if (delta < tick_period) 0.00 : ffff80001017b5b8: ldr x1, [x21, #968] 0.00 : ffff80001017b5bc: cmp x0, x1 0.00 : ffff80001017b5c0: b.lt ffff80001017b5cc <__tick_nohz_idle_restart_tick+0x3c> // b.tstop 0.00 : ffff80001017b5c4: mov x0, x20 0.00 : ffff80001017b5c8: bl ffff80001017b488 : tick_nohz_restart_sched_tick(): : timer_clear_idle(); 1.74 : ffff80001017b5cc: bl ffff80001016a9b0 : calc_load_nohz_stop(); 46.06 : ffff80001017b5d0: bl ffff80001011ae08 : ts->idle_exittime = now; 9.01 : ffff80001017b5d4: str x20, [x19, #136] : ts->tick_stopped = 0; 0.92 : ffff80001017b5d8: ldrb w1, [x19, #76] : tick_nohz_restart(): : hrtimer_cancel(&ts->sched_timer); 0.00 : ffff80001017b5dc: mov x0, x19 : tick_nohz_restart_sched_tick(): : ts->tick_stopped = 0; 0.00 : ffff80001017b5e0: and w1, w1, #0xfffffffd 0.00 : ffff80001017b5e4: strb w1, [x19, #76] : tick_nohz_restart(): : hrtimer_cancel(&ts->sched_timer); 0.00 : ffff80001017b5e8: bl ffff80001016bad0 : hrtimer_forward(&ts->sched_timer, now, tick_period); 0.00 : ffff80001017b5ec: ldr x2, [x21, #968] 0.00 : ffff80001017b5f0: mov x1, x20 : hrtimer_set_expires(&ts->sched_timer, ts->last_tick); 0.89 : ffff80001017b5f4: ldr x0, [x19, #80] : hrtimer_set_expires(): : } ____cacheline_aligned; : : static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) : { : timer->node.expires = time; : timer->_softexpires = time; 0.94 : ffff80001017b5f8: stp x0, x0, [x19, #24] : tick_nohz_restart(): : hrtimer_forward(&ts->sched_timer, now, tick_period); 0.00 : ffff80001017b5fc: mov x0, x19 0.00 : ffff80001017b600: bl ffff80001016acf8 : if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { 0.00 : ffff80001017b604: ldr w0, [x19, #72] 0.00 : ffff80001017b608: cmp w0, #0x2 0.00 : ffff80001017b60c: b.eq ffff80001017b658 <__tick_nohz_idle_restart_tick+0xc8> // b.none : tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); 0.00 : ffff80001017b610: ldr x0, [x19, #24] 0.00 : ffff80001017b614: mov w1, #0x1 // #1 0.00 : ffff80001017b618: bl ffff80001017afa8 : tick_nohz_account_idle_ticks(): : ticks = jiffies - ts->idle_jiffies; 0.00 : ffff80001017b61c: adrp x0, ffff800011907000 0.00 : ffff80001017b620: ldr x2, [x19, #96] : tick_nohz_restart(): : ts->next_tick = 0; 0.00 : ffff80001017b624: str xzr, [x19, #88] : tick_nohz_account_idle_ticks(): : if (ticks && ticks < LONG_MAX) 0.00 : ffff80001017b628: mov x1, #0xfffffffffffffffd // #-3 : ticks = jiffies - ts->idle_jiffies; 0.00 : ffff80001017b62c: ldr x0, [x0, #2432] : if (ticks && ticks < LONG_MAX) 0.00 : ffff80001017b630: movk x1, #0x7fff, lsl #48 : ticks = jiffies - ts->idle_jiffies; 0.00 : ffff80001017b634: sub x0, x0, x2 : if (ticks && ticks < LONG_MAX) 0.00 : ffff80001017b638: sub x2, x0, #0x1 0.00 : ffff80001017b63c: cmp x2, x1 0.00 : ffff80001017b640: b.hi ffff80001017b648 <__tick_nohz_idle_restart_tick+0xb8> // b.pmore : account_idle_ticks(ticks); 0.00 : ffff80001017b644: bl ffff80001011ba88 : __tick_nohz_idle_restart_tick(): : tick_nohz_restart_sched_tick(ts, now); : tick_nohz_account_idle_ticks(ts); : } 0.00 : ffff80001017b648: ldp x19, x20, [sp, #16] 3.65 : ffff80001017b64c: ldr x21, [sp, #32] 0.00 : ffff80001017b650: ldp x29, x30, [sp], #48 0.00 : ffff80001017b654: ret : hrtimer_start_expires(): : static inline void hrtimer_start_expires(struct hrtimer *timer, : enum hrtimer_mode mode) : { : u64 delta; : ktime_t soft, hard; : soft = hrtimer_get_softexpires(timer); 0.00 : ffff80001017b658: ldp x2, x1, [x19, #24] : hard = hrtimer_get_expires(timer); : delta = ktime_to_ns(ktime_sub(hard, soft)); : hrtimer_start_range_ns(timer, soft, delta, mode); 0.00 : ffff80001017b65c: mov w3, #0xa // #10 0.00 : ffff80001017b660: mov x0, x19 0.00 : ffff80001017b664: sub x2, x2, x1 26.74 : ffff80001017b668: bl ffff80001016b750 0.00 : ffff80001017b66c: b ffff80001017b61c <__tick_nohz_idle_restart_tick+0x8c> Percent | Source code & Disassembly of vmlinux for cycles (106 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011e188 : : nohz_balance_exit_idle(): : atomic_inc(&sd->shared->nr_busy_cpus); : unlock: : rcu_read_unlock(); : } : : void nohz_balance_exit_idle(struct rq *rq) 0.00 : ffff80001011e188: stp x29, x30, [sp, #-32]! : clear_bit(): : atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); : } : : static inline void clear_bit(unsigned int nr, volatile unsigned long *p) : { : p += BIT_WORD(nr); 0.00 : ffff80001011e18c: adrp x1, ffff800011afb000 : atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff80001011e190: mov x3, #0x1 // #1 : p += BIT_WORD(nr); 0.00 : ffff80001011e194: add x5, x1, #0xfc0 : nohz_balance_exit_idle(): 0.00 : ffff80001011e198: mov x29, sp 0.00 : ffff80001011e19c: str x19, [sp, #16] : SCHED_WARN_ON(rq != this_rq()); : : if (likely(!rq->nohz_tick_stopped)) : return; : : rq->nohz_tick_stopped = 0; 0.00 : ffff80001011e1a0: str wzr, [x0, #44] : cpumask_clear_cpu(): : * @cpu: cpu number (< nr_cpu_ids) : * @dstp: the cpumask pointer : */ : static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) : { : clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 0.00 : ffff80001011e1a4: ldr w4, [x0, #2568] : clear_bit(): 0.00 : ffff80001011e1a8: lsr w2, w4, #6 : atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff80001011e1ac: lsl x3, x3, x4 : p += BIT_WORD(nr); 0.00 : ffff80001011e1b0: add x2, x5, x2, lsl #3 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011e1b4: b ffff80001011e1e0 0.00 : ffff80001011e1b8: b ffff80001011e1e0 : __lse_atomic64_andnot(): : " " #asm_op " %[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) 0.00 : ffff80001011e1bc: stclr x3, [x2] : arch_static_branch_jump(): 0.00 : ffff80001011e1c0: b ffff80001011e1e8 0.00 : ffff80001011e1c4: b ffff80001011e1e8 : __lse_atomic_sub(): : asm volatile( 0.00 : ffff80001011e1c8: add x1, x1, #0xfc0 0.00 : ffff80001011e1cc: mov w2, #0x1 // #1 0.00 : ffff80001011e1d0: add x3, x1, #0x20 0.00 : ffff80001011e1d4: neg w2, w2 0.00 : ffff80001011e1d8: stadd w2, [x3] 65.96 : ffff80001011e1dc: b ffff80001011e1f8 : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff80001011e1e0: b ffff80001012890c 0.00 : ffff80001011e1e4: b ffff80001011e1c0 : __ll_sc_atomic_sub(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff80001011e1e8: add x1, x1, #0xfc0 0.00 : ffff80001011e1ec: mov w2, #0x1 // #1 0.00 : ffff80001011e1f0: add x1, x1, #0x20 0.00 : ffff80001011e1f4: b ffff800010128924 : nohz_balance_exit_idle(): : cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); : atomic_dec(&nohz.nr_cpus); : : set_cpu_sd_state_busy(rq->cpu); 0.00 : ffff80001011e1f8: ldr w19, [x0, #2568] : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001011e1fc: bl ffff80001015c1b0 <__rcu_read_lock> : set_cpu_sd_state_busy(): : sd = rcu_dereference(per_cpu(sd_llc, cpu)); 0.00 : ffff80001011e200: adrp x1, ffff800011909000 0.00 : ffff80001011e204: add x1, x1, #0x928 0.00 : ffff80001011e208: adrp x0, ffff80001151f000 0.00 : ffff80001011e20c: add x0, x0, #0x118 0.00 : ffff80001011e210: ldr x1, [x1, w19, sxtw #3] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011e214: ldr x0, [x0, x1] : set_cpu_sd_state_busy(): : if (!sd || !sd->nohz_idle) 0.00 : ffff80001011e218: cbz x0, ffff80001011e224 0.00 : ffff80001011e21c: ldr w1, [x0, #52] 0.00 : ffff80001011e220: cbnz w1, ffff80001011e234 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff80001011e224: bl ffff800010160d20 <__rcu_read_unlock> : nohz_balance_exit_idle(): : } 0.00 : ffff80001011e228: ldr x19, [sp, #16] 0.00 : ffff80001011e22c: ldp x29, x30, [sp], #32 0.00 : ffff80001011e230: ret : set_cpu_sd_state_busy(): : atomic_inc(&sd->shared->nr_busy_cpus); 0.00 : ffff80001011e234: ldr x1, [x0, #120] : sd->nohz_idle = 0; 0.00 : ffff80001011e238: str wzr, [x0, #52] : atomic_inc(&sd->shared->nr_busy_cpus); 0.00 : ffff80001011e23c: add x2, x1, #0x4 : arch_static_branch_jump(): 0.00 : ffff80001011e240: b ffff80001011e260 0.00 : ffff80001011e244: b ffff80001011e260 : __lse_atomic_add(): : ATOMIC_OP(add, stadd) 0.00 : ffff80001011e248: mov w0, #0x1 // #1 0.00 : ffff80001011e24c: stadd w0, [x2] : rcu_read_unlock(): 34.04 : ffff80001011e250: bl ffff800010160d20 <__rcu_read_unlock> : nohz_balance_exit_idle(): : } 0.00 : ffff80001011e254: ldr x19, [sp, #16] 0.00 : ffff80001011e258: ldp x29, x30, [sp], #32 0.00 : ffff80001011e25c: ret : __ll_sc_atomic_add(): : ATOMIC_OPS(add, add, I) 0.00 : ffff80001011e260: add x1, x1, #0x4 0.00 : ffff80001011e264: b ffff80001012893c : rcu_read_unlock(): 0.00 : ffff80001011e268: bl ffff800010160d20 <__rcu_read_unlock> : nohz_balance_exit_idle(): 0.00 : ffff80001011e26c: ldr x19, [sp, #16] 0.00 : ffff80001011e270: ldp x29, x30, [sp], #32 0.00 : ffff80001011e274: ret Percent | Source code & Disassembly of vmlinux for cycles (106 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012aec8 : : dequeue_task_rt(): : if (!task_current(rq, p) && p->nr_cpus_allowed > 1) : enqueue_pushable_task(rq, p); : } : : static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) : { 52.11 : ffff80001012aec8: stp x29, x30, [sp, #-48]! 0.00 : ffff80001012aecc: mov x29, sp 24.16 : ffff80001012aed0: stp x21, x22, [sp, #32] 0.00 : ffff80001012aed4: mov x21, x1 3.72 : ffff80001012aed8: stp x19, x20, [sp, #16] 0.00 : ffff80001012aedc: mov w19, w2 0.00 : ffff80001012aee0: mov x20, x0 : struct sched_rt_entity *rt_se = &p->rt; : : update_curr_rt(rq); 0.00 : ffff80001012aee4: bl ffff80001012aab8 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001012aee8: ldr w4, [x21, #68] : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012aeec: adrp x3, ffff800011909000 0.00 : ffff80001012aef0: add x3, x3, #0x928 : dequeue_rt_entity(): : dequeue_rt_stack(rt_se, flags); 0.00 : ffff80001012aef4: mov w1, w19 4.85 : ffff80001012aef8: add x0, x21, #0x1c0 : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012aefc: adrp x2, ffff800011528000 0.00 : ffff80001012af00: add x2, x2, #0xe80 : dequeue_pushable_task(): : plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 0.00 : ffff80001012af04: add x22, x20, #0x878 : rq_of_rt_se(): : return task_rq(p); 2.79 : ffff80001012af08: ldr x19, [x3, w4, uxtw #3] 0.00 : ffff80001012af0c: add x19, x2, x19 : dequeue_rt_entity(): : dequeue_rt_stack(rt_se, flags); 0.00 : ffff80001012af10: bl ffff80001012a5a0 : enqueue_top_rt_rq(&rq->rt); 0.00 : ffff80001012af14: add x0, x19, #0x200 0.00 : ffff80001012af18: bl ffff8000101292d0 : dequeue_pushable_task(): : plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 1.95 : ffff80001012af1c: add x0, x21, #0x378 0.00 : ffff80001012af20: mov x1, x22 0.00 : ffff80001012af24: bl ffff800010cc2748 : __read_once_size(): 0.00 : ffff80001012af28: ldr x0, [x20, #2168] : dequeue_pushable_task(): : if (has_pushable_tasks(rq)) { 0.00 : ffff80001012af2c: cmp x22, x0 0.00 : ffff80001012af30: b.eq ffff80001012af54 // b.none : rq->rt.highest_prio.next = p->prio; 0.00 : ffff80001012af34: ldr x0, [x20, #2168] 0.00 : ffff80001012af38: sub x0, x0, #0x390 0.00 : ffff80001012af3c: ldr w0, [x0, #108] 0.00 : ffff80001012af40: str w0, [x20, #2140] : dequeue_task_rt(): : dequeue_rt_entity(rt_se, flags); : : dequeue_pushable_task(rq, p); : } 0.00 : ffff80001012af44: ldp x19, x20, [sp, #16] 0.00 : ffff80001012af48: ldp x21, x22, [sp, #32] 0.00 : ffff80001012af4c: ldp x29, x30, [sp], #48 0.00 : ffff80001012af50: ret : dequeue_pushable_task(): : rq->rt.highest_prio.next = MAX_RT_PRIO; 0.00 : ffff80001012af54: mov w0, #0x64 // #100 1.78 : ffff80001012af58: str w0, [x20, #2140] : dequeue_task_rt(): : } 8.65 : ffff80001012af5c: ldp x19, x20, [sp, #16] 0.00 : ffff80001012af60: ldp x21, x22, [sp, #32] 0.00 : ffff80001012af64: ldp x29, x30, [sp], #48 0.00 : ffff80001012af68: ret Percent | Source code & Disassembly of vmlinux for cycles (104 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017a4a0 : : tick_check_oneshot_broadcast_this_cpu(): : /* : * Called from irq_enter() when idle was interrupted to reenable the : * per cpu device. : */ : void tick_check_oneshot_broadcast_this_cpu(void) : { 4.83 : ffff80001017a4a0: stp x29, x30, [sp, #-16]! : if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) { 0.00 : ffff80001017a4a4: adrp x0, ffff80001151d000 0.00 : ffff80001017a4a8: add x0, x0, #0x18 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001017a4ac: adrp x2, ffff800011b22000 : tick_check_oneshot_broadcast_this_cpu(): : { 6.81 : ffff80001017a4b0: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001017a4b4: mrs x3, tpidr_el1 : tick_check_oneshot_broadcast_this_cpu(): : if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) { 0.00 : ffff80001017a4b8: ldr w1, [x0, x3] : test_bit(): 0.00 : ffff80001017a4bc: add x2, x2, #0x3d8 0.00 : ffff80001017a4c0: add w0, w1, #0x3f 0.00 : ffff80001017a4c4: cmp w1, #0x0 31.86 : ffff80001017a4c8: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff80001017a4cc: asr w0, w0, #6 0.00 : ffff80001017a4d0: sxtw x0, w0 0.00 : ffff80001017a4d4: ldr x0, [x2, x0, lsl #3] 0.00 : ffff80001017a4d8: lsr x1, x0, x1 : tick_check_oneshot_broadcast_this_cpu(): 0.00 : ffff80001017a4dc: tbz w1, #0, ffff80001017a4f8 : struct tick_device *td = this_cpu_ptr(&tick_cpu_device); 0.00 : ffff80001017a4e0: adrp x0, ffff800011525000 0.00 : ffff80001017a4e4: add x0, x0, #0xec0 0.00 : ffff80001017a4e8: add x1, x0, x3 : /* : * We might be in the middle of switching over from : * periodic to oneshot. If the CPU has not yet : * switched over, leave the device alone. : */ : if (td->mode == TICKDEV_MODE_ONESHOT) { 0.00 : ffff80001017a4ec: ldr w1, [x1, #8] 0.00 : ffff80001017a4f0: cmp w1, #0x1 0.00 : ffff80001017a4f4: b.eq ffff80001017a500 // b.none : clockevents_switch_state(td->evtdev, : CLOCK_EVT_STATE_ONESHOT); : } : } : } 56.49 : ffff80001017a4f8: ldp x29, x30, [sp], #16 0.00 : ffff80001017a4fc: ret : clockevents_switch_state(td->evtdev, 0.00 : ffff80001017a500: ldr x0, [x0, x3] 0.00 : ffff80001017a504: mov w1, #0x3 // #3 0.00 : ffff80001017a508: bl ffff8000101781e0 : } 0.00 : ffff80001017a50c: ldp x29, x30, [sp], #16 0.00 : ffff80001017a510: ret Percent | Source code & Disassembly of vmlinux for cycles (170 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010085378 <__get_cpu_fpsimd_context>: : __get_cpu_fpsimd_context(): : DEFINE_PER_CPU(bool, fpsimd_context_busy); : EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy); : : static void __get_cpu_fpsimd_context(void) : { : bool busy = __this_cpu_xchg(fpsimd_context_busy, true); 10.16 : ffff800010085378: adrp x0, ffff80001151d000 0.00 : ffff80001008537c: add x0, x0, #0x60 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.65 : ffff800010085380: mrs x1, tpidr_el1 : __get_cpu_fpsimd_context(): 0.57 : ffff800010085384: add x0, x0, #0x18 0.00 : ffff800010085388: mov w3, #0x1 // #1 0.00 : ffff80001008538c: ldrb w2, [x0, x1] 88.02 : ffff800010085390: strb w3, [x0, x1] : : WARN_ON(busy); 0.00 : ffff800010085394: cbnz w2, ffff80001008539c <__get_cpu_fpsimd_context+0x24> : } 0.59 : ffff800010085398: ret : WARN_ON(busy); 0.00 : ffff80001008539c: brk #0x800 : } 0.00 : ffff8000100853a0: ret Percent | Source code & Disassembly of vmlinux for cycles (102 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012a2d0 : : pull_rt_task(): : irq_work_queue_on(&rd->rto_push_work, cpu); : } : #endif /* HAVE_RT_PUSH_IPI */ : : static void pull_rt_task(struct rq *this_rq) : { 0.00 : ffff80001012a2d0: stp x29, x30, [sp, #-48]! 0.00 : ffff80001012a2d4: mov x29, sp 0.94 : ffff80001012a2d8: str x19, [sp, #16] 0.00 : ffff80001012a2dc: mov x19, x0 : int this_cpu = this_rq->cpu, cpu; : bool resched = false; : struct task_struct *p; : struct rq *src_rq; : int rt_overload_count = rt_overloaded(this_rq); 2.02 : ffff80001012a2e0: ldr x0, [x0, #2464] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001012a2e4: ldr w0, [x0, #4] : pull_rt_task(): : : if (likely(!rt_overload_count)) 0.00 : ffff80001012a2e8: cbnz w0, ffff80001012a2f8 : double_unlock_balance(this_rq, src_rq); : } : : if (resched) : resched_curr(this_rq); : } 57.51 : ffff80001012a2ec: ldr x19, [sp, #16] 39.53 : ffff80001012a2f0: ldp x29, x30, [sp], #48 0.00 : ffff80001012a2f4: ret : smp_rmb(); 0.00 : ffff80001012a2f8: dmb ishld : if (rt_overload_count == 1 && 0.00 : ffff80001012a2fc: cmp w0, #0x1 0.00 : ffff80001012a300: b.ne ffff80001012a398 // b.any : cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask)) 0.00 : ffff80001012a304: ldr w2, [x19, #2568] 0.00 : ffff80001012a308: ldr x0, [x19, #2464] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001012a30c: add w1, w2, #0x3f 0.00 : ffff80001012a310: cmp w2, #0x0 0.00 : ffff80001012a314: csel w1, w1, w2, lt // lt = tstop : cpumask_test_cpu(): : * : * Returns 1 if @cpu is set in @cpumask, else returns 0 : */ : static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask) : { : return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); 0.00 : ffff80001012a318: add x4, x0, #0x100 : test_bit(): 0.00 : ffff80001012a31c: asr w1, w1, #6 0.00 : ffff80001012a320: sxtw x1, w1 0.00 : ffff80001012a324: ldr x1, [x4, x1, lsl #3] 0.00 : ffff80001012a328: lsr x2, x1, x2 : pull_rt_task(): : if (rt_overload_count == 1 && 0.00 : ffff80001012a32c: tbnz w2, #0, ffff80001012a2ec 0.00 : ffff80001012a330: str x20, [x29, #24] : tell_cpu_to_push(): : atomic_inc(&rq->rd->rto_loop_next); 0.00 : ffff80001012a334: add x0, x0, #0xf4 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001012a338: b ffff80001012a390 0.00 : ffff80001012a33c: b ffff80001012a390 : __lse_atomic_add(): : } : : ATOMIC_OP(andnot, stclr) : ATOMIC_OP(or, stset) : ATOMIC_OP(xor, steor) : ATOMIC_OP(add, stadd) 0.00 : ffff80001012a340: mov w1, #0x1 // #1 0.00 : ffff80001012a344: stadd w1, [x0] : atomic_cmpxchg_acquire(): : #if defined(arch_atomic_cmpxchg_acquire) : static inline int : atomic_cmpxchg_acquire(atomic_t *v, int old, int new) : { : kasan_check_write(v, sizeof(*v)); : return arch_atomic_cmpxchg_acquire(v, old, new); 0.00 : ffff80001012a348: ldr x4, [x19, #2464] 0.00 : ffff80001012a34c: add x0, x4, #0xf8 : arch_static_branch_jump(): 0.00 : ffff80001012a350: b ffff80001012a37c 0.00 : ffff80001012a354: b ffff80001012a37c : __lse__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, ) : __CMPXCHG_CASE(w, , , 32, ) : __CMPXCHG_CASE(x, , , 64, ) : __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") : __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 0.00 : ffff80001012a358: mov w1, #0x0 // #0 0.00 : ffff80001012a35c: mov w2, #0x1 // #1 0.00 : ffff80001012a360: mov w4, w1 0.00 : ffff80001012a364: casa w4, w2, [x0] 0.00 : ffff80001012a368: mov w0, w4 0.00 : ffff80001012a36c: mov w20, w0 : tell_cpu_to_push(): : if (!rto_start_trylock(&rq->rd->rto_loop_start)) 0.00 : ffff80001012a370: cbz w20, ffff80001012a3a4 0.00 : ffff80001012a374: ldr x20, [x29, #24] 0.00 : ffff80001012a378: b ffff80001012a2ec : __ll_sc__cmpxchg_case_acq_32(): : __CMPXCHG_CASE(w, h, , 16, , , , , K) : __CMPXCHG_CASE(w, , , 32, , , , , K) : __CMPXCHG_CASE( , , , 64, , , , , L) : __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) : __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 0.00 : ffff80001012a37c: mov x0, #0x0 // #0 0.00 : ffff80001012a380: mov w1, #0x1 // #1 0.00 : ffff80001012a384: add x3, x4, #0xf8 0.00 : ffff80001012a388: b ffff80001012b438 0.00 : ffff80001012a38c: b ffff80001012a370 : __ll_sc_atomic_add(): : ATOMIC_OPS(add, add, I) 0.00 : ffff80001012a390: b ffff80001012b454 0.00 : ffff80001012a394: b ffff80001012a348 0.00 : ffff80001012a398: str x20, [x29, #24] 0.00 : ffff80001012a39c: ldr x0, [x19, #2464] 0.00 : ffff80001012a3a0: b ffff80001012a334 : tell_cpu_to_push(): : raw_spin_lock(&rq->rd->rto_lock); 0.00 : ffff80001012a3a4: ldr x0, [x19, #2464] 0.00 : ffff80001012a3a8: add x0, x0, #0xe8 0.00 : ffff80001012a3ac: bl ffff800010cd80f0 <_raw_spin_lock> : if (rq->rd->rto_cpu < 0) 0.00 : ffff80001012a3b0: ldr x0, [x19, #2464] 0.00 : ffff80001012a3b4: ldr w1, [x0, #240] 0.00 : ffff80001012a3b8: tbnz w1, #31, ffff80001012a3d8 : raw_spin_unlock(&rq->rd->rto_lock); 0.00 : ffff80001012a3bc: add x0, x0, #0xe8 0.00 : ffff80001012a3c0: bl ffff800010cd7c38 <_raw_spin_unlock> : atomic_set_release(): : : #ifndef atomic_set_release : static inline void : atomic_set_release(atomic_t *v, int i) : { : smp_store_release(&(v)->counter, i); 0.00 : ffff80001012a3c4: ldr x0, [x19, #2464] 0.00 : ffff80001012a3c8: add x0, x0, #0xf8 0.00 : ffff80001012a3cc: stlr w20, [x0] 0.00 : ffff80001012a3d0: ldr x20, [x29, #24] 0.00 : ffff80001012a3d4: b ffff80001012a2ec 0.00 : ffff80001012a3d8: str x21, [x29, #32] : tell_cpu_to_push(): : cpu = rto_next_cpu(rq->rd); 0.00 : ffff80001012a3dc: bl ffff800010128c28 0.00 : ffff80001012a3e0: mov w21, w0 : raw_spin_unlock(&rq->rd->rto_lock); 0.00 : ffff80001012a3e4: ldr x0, [x19, #2464] 0.00 : ffff80001012a3e8: add x0, x0, #0xe8 0.00 : ffff80001012a3ec: bl ffff800010cd7c38 <_raw_spin_unlock> : atomic_set_release(): 0.00 : ffff80001012a3f0: ldr x0, [x19, #2464] 0.00 : ffff80001012a3f4: add x0, x0, #0xf8 0.00 : ffff80001012a3f8: stlr w20, [x0] : tell_cpu_to_push(): : if (cpu >= 0) { 0.00 : ffff80001012a3fc: tbnz w21, #31, ffff80001012a420 : sched_get_rd(rq->rd); 0.00 : ffff80001012a400: ldr x0, [x19, #2464] 0.00 : ffff80001012a404: bl ffff800010133000 : irq_work_queue_on(&rq->rd->rto_push_work, cpu); 0.00 : ffff80001012a408: ldr x0, [x19, #2464] 0.00 : ffff80001012a40c: mov w1, w21 0.00 : ffff80001012a410: add x0, x0, #0xd0 0.00 : ffff80001012a414: bl ffff8000101b4700 0.00 : ffff80001012a418: ldp x20, x21, [x29, #24] 0.00 : ffff80001012a41c: b ffff80001012a2ec 0.00 : ffff80001012a420: ldp x20, x21, [x29, #24] 0.00 : ffff80001012a424: b ffff80001012a2ec Percent | Source code & Disassembly of vmlinux for cycles (312 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d90f8 : : aio_read_events(): : return ret; : } : : static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr, : struct io_event __user *event, long *i) : { 0.32 : ffff8000102d90f8: stp x29, x30, [sp, #-128]! 0.00 : ffff8000102d90fc: mov x29, sp 6.46 : ffff8000102d9100: stp x19, x20, [sp, #16] 0.00 : ffff8000102d9104: mov x20, x4 1.29 : ffff8000102d9108: stp x21, x22, [sp, #32] : aio_read_events_ring(): : mutex_lock(&ctx->ring_lock); 0.00 : ffff8000102d910c: add x22, x0, #0x180 : aio_read_events(): : { 0.00 : ffff8000102d9110: stp x23, x24, [sp, #48] 0.00 : ffff8000102d9114: mov x21, x3 0.97 : ffff8000102d9118: str x26, [sp, #72] 0.00 : ffff8000102d911c: mov x23, x2 0.28 : ffff8000102d9120: str x28, [sp, #88] 0.00 : ffff8000102d9124: mov x28, x0 0.95 : ffff8000102d9128: str x1, [x29, #104] : aio_read_events_ring(): : mutex_lock(&ctx->ring_lock); 0.00 : ffff8000102d912c: mov x0, x22 : aio_read_events(): : long ret = aio_read_events_ring(ctx, event + *i, nr - *i); 0.98 : ffff8000102d9130: ldr x19, [x4] : aio_read_events_ring(): : mutex_lock(&ctx->ring_lock); 0.00 : ffff8000102d9134: bl ffff800010cd4b28 : ring = kmap_atomic(ctx->ring_pages[0]); 3.24 : ffff8000102d9138: ldr x1, [x28, #168] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.97 : ffff8000102d913c: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000102d9140: ldr w2, [x0, #16] : aio_read_events_ring(): 2.54 : ffff8000102d9144: ldr x1, [x1] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000102d9148: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102d914c: str w2, [x0, #16] : pagefault_disabled_inc(): : } : #endif : : static __always_inline void pagefault_disabled_inc(void) : { : current->pagefault_disabled++; 0.00 : ffff8000102d9150: ldr w2, [x0, #2448] 0.00 : ffff8000102d9154: add w2, w2, #0x1 1.61 : ffff8000102d9158: str w2, [x0, #2448] : lowmem_page_address(): : */ : #include : : static __always_inline void *lowmem_page_address(const struct page *page) : { : return page_to_virt(page); 0.00 : ffff8000102d915c: mov x2, #0x200000 // #2097152 0.00 : ffff8000102d9160: movk x2, #0x200, lsl #32 0.00 : ffff8000102d9164: add x1, x1, x2 0.00 : ffff8000102d9168: mov x2, #0xffff000000000000 // #-281474976710656 0.00 : ffff8000102d916c: lsr x1, x1, #6 0.00 : ffff8000102d9170: add x1, x2, x1, lsl #12 : aio_read_events_ring(): : tail = ring->tail; 0.32 : ffff8000102d9174: ldp w26, w24, [x1, #8] : pagefault_disabled_dec(): : } : : static __always_inline void pagefault_disabled_dec(void) : { : current->pagefault_disabled--; 23.13 : ffff8000102d9178: ldr w1, [x0, #2448] 0.00 : ffff8000102d917c: sub w1, w1, #0x1 1.60 : ffff8000102d9180: str w1, [x0, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.65 : ffff8000102d9184: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102d9188: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.32 : ffff8000102d918c: str w1, [x0, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102d9190: cbz x1, ffff8000102d9400 : __read_once_size(): : __READ_ONCE_SIZE; 0.65 : ffff8000102d9194: ldr x0, [x0, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102d9198: cbz x0, ffff8000102d9400 : aio_read_events_ring(): : smp_rmb(); 1.54 : ffff8000102d919c: dmb ishld : if (head == tail) 21.16 : ffff8000102d91a0: cmp w26, w24 0.00 : ffff8000102d91a4: b.eq ffff8000102d9410 // b.none 0.00 : ffff8000102d91a8: str x27, [x29, #80] : aio_read_events(): : long ret = aio_read_events_ring(ctx, event + *i, nr - *i); 0.00 : ffff8000102d91ac: sub x5, x23, x19 : aio_read_events_ring(): : while (ret < nr) { 0.00 : ffff8000102d91b0: cmp x5, #0x0 : head %= ctx->nr_events; 0.00 : ffff8000102d91b4: ldr w0, [x28, #144] 0.00 : ffff8000102d91b8: udiv w1, w26, w0 : tail %= ctx->nr_events; 0.00 : ffff8000102d91bc: udiv w6, w24, w0 : head %= ctx->nr_events; 0.00 : ffff8000102d91c0: msub w26, w1, w0, w26 : tail %= ctx->nr_events; 0.00 : ffff8000102d91c4: msub w6, w6, w0, w24 : while (ret < nr) { 0.00 : ffff8000102d91c8: b.le ffff8000102d946c : aio_read_events(): : long ret = aio_read_events_ring(ctx, event + *i, nr - *i); 0.65 : ffff8000102d91cc: lsl x24, x19, #5 0.00 : ffff8000102d91d0: str x25, [x29, #64] : aio_read_events_ring(): : long ret = 0; 0.00 : ffff8000102d91d4: mov x27, #0x0 // #0 : avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); 0.00 : ffff8000102d91d8: mov x23, #0x80 // #128 : check_copy_size(): : __bad_copy_from(); : else : __bad_copy_to(); : return false; : } : if (WARN_ON_ONCE(bytes > INT_MAX)) 0.00 : ffff8000102d91dc: mov x25, #0x7fffffff // #2147483647 : aio_read_events_ring(): : avail = (head <= tail ? tail : ctx->nr_events) - head; 0.00 : ffff8000102d91e0: sub w0, w0, w26 0.00 : ffff8000102d91e4: cmp w6, w26 0.00 : ffff8000102d91e8: b.cc ffff8000102d91f4 // b.lo, b.ul, b.last 0.00 : ffff8000102d91ec: sub w0, w6, w26 : if (head == tail) 0.00 : ffff8000102d91f0: b.eq ffff8000102d9284 // b.none : pos = head + AIO_EVENTS_OFFSET; 0.97 : ffff8000102d91f4: add w1, w26, #0x1 : avail = min(avail, nr - ret); 0.00 : ffff8000102d91f8: sub x19, x5, x27 : avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); 0.00 : ffff8000102d91fc: and x10, x1, #0x7f : copy_ret = copy_to_user(event + ret, ev + pos, 0.00 : ffff8000102d9200: add x4, x24, x27, lsl #5 : avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); 0.00 : ffff8000102d9204: sub x2, x23, x10 : copy_ret = copy_to_user(event + ret, ev + pos, 0.00 : ffff8000102d9208: add x4, x21, x4 : avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); 0.00 : ffff8000102d920c: cmp x19, x2 0.00 : ffff8000102d9210: csel x19, x19, x2, le 0.00 : ffff8000102d9214: cmp x19, x0 0.00 : ffff8000102d9218: csel x19, x19, x0, le : copy_ret = copy_to_user(event + ret, ev + pos, 0.00 : ffff8000102d921c: lsl x2, x19, #5 : check_copy_size(): 0.00 : ffff8000102d9220: cmp x2, x25 0.00 : ffff8000102d9224: b.hi ffff8000102d9428 // b.pmore : get_current(): 0.00 : ffff8000102d9228: mrs x3, sp_el0 : __range_ok(): : * Asynchronous I/O running in a kernel thread does not have the : * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag : * the user address before checking. : */ : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && : (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) 0.00 : ffff8000102d922c: ldr w11, [x3, #44] : unsigned long ret, limit = current_thread_info()->addr_limit; 0.29 : ffff8000102d9230: ldr x0, [x3, #8] : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 0.00 : ffff8000102d9234: tbnz w11, #21, ffff8000102d9244 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.65 : ffff8000102d9238: ldr x11, [x3] : __range_ok(): 0.00 : ffff8000102d923c: mov x3, x4 0.65 : ffff8000102d9240: tbz w11, #26, ffff8000102d924c : sign_extend64(): : * @index: 0 based bit index (0<=index<64) to sign bit : */ : static inline __s64 sign_extend64(__u64 value, int index) : { : __u8 shift = 63 - index; : return (__s64)(value << shift) >> shift; 0.00 : ffff8000102d9244: sbfx x3, x4, #0, #56 : __range_ok(): : addr = untagged_addr(addr); 0.00 : ffff8000102d9248: and x3, x4, x3 : : __chk_user_ptr(addr); : asm volatile( 0.00 : ffff8000102d924c: adds x3, x3, x2 0.00 : ffff8000102d9250: csel x0, xzr, x0, hi // hi = pmore 0.00 : ffff8000102d9254: csinv x3, x3, xzr, cc // cc = lo, ul, last 0.00 : ffff8000102d9258: sbcs xzr, x3, x0 0.00 : ffff8000102d925c: cset x3, ls // ls = plast : _copy_to_user(): : if (access_ok(to, n)) { 0.00 : ffff8000102d9260: cbnz x3, ffff8000102d9350 : aio_read_events_ring(): : if (unlikely(copy_ret)) { 0.00 : ffff8000102d9264: cbnz w2, ffff8000102d93b8 : head %= ctx->nr_events; 0.65 : ffff8000102d9268: ldr w0, [x28, #144] : head += avail; 0.00 : ffff8000102d926c: add w26, w26, w19 : ret += avail; 0.00 : ffff8000102d9270: add x27, x27, x19 : while (ret < nr) { 0.00 : ffff8000102d9274: cmp x5, x27 : head %= ctx->nr_events; 0.00 : ffff8000102d9278: udiv w1, w26, w0 0.00 : ffff8000102d927c: msub w26, w1, w0, w26 : while (ret < nr) { 0.00 : ffff8000102d9280: b.gt ffff8000102d91e0 0.65 : ffff8000102d9284: ldr x25, [x29, #64] : ring = kmap_atomic(ctx->ring_pages[0]); 0.65 : ffff8000102d9288: ldr x1, [x28, #168] : get_current(): 0.00 : ffff8000102d928c: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff8000102d9290: ldr w2, [x0, #16] : aio_read_events_ring(): 0.98 : ffff8000102d9294: ldr x1, [x1] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102d9298: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.62 : ffff8000102d929c: str w2, [x0, #16] : pagefault_disabled_inc(): : current->pagefault_disabled++; 0.29 : ffff8000102d92a0: ldr w2, [x0, #2448] 0.00 : ffff8000102d92a4: add w2, w2, #0x1 0.29 : ffff8000102d92a8: str w2, [x0, #2448] : lowmem_page_address(): 0.00 : ffff8000102d92ac: mov x2, #0x200000 // #2097152 0.00 : ffff8000102d92b0: movk x2, #0x200, lsl #32 0.00 : ffff8000102d92b4: add x1, x1, x2 : aio_read_events_ring(): : ring->head = head; 0.65 : ffff8000102d92b8: mov x2, #0xffff000000000000 // #-281474976710656 : lowmem_page_address(): 0.00 : ffff8000102d92bc: lsr x1, x1, #6 : aio_read_events_ring(): 0.30 : ffff8000102d92c0: add x1, x2, x1, lsl #12 0.00 : ffff8000102d92c4: str w26, [x1, #8] : pagefault_disabled_dec(): : current->pagefault_disabled--; 1.62 : ffff8000102d92c8: ldr w1, [x0, #2448] 0.00 : ffff8000102d92cc: sub w1, w1, #0x1 0.64 : ffff8000102d92d0: str w1, [x0, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102d92d4: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102d92d8: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.66 : ffff8000102d92dc: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102d92e0: cbnz x1, ffff8000102d9430 : __kunmap_atomic(): : #define kmap_atomic_prot(page, prot) kmap_atomic(page) : : static inline void __kunmap_atomic(void *addr) : { : pagefault_enable(); : preempt_enable(); 0.00 : ffff8000102d92e4: bl ffff800010cd2e50 : aio_read_events_ring(): : flush_dcache_page(ctx->ring_pages[0]); 0.32 : ffff8000102d92e8: ldr x0, [x28, #168] 0.97 : ffff8000102d92ec: ldr x0, [x0] 0.00 : ffff8000102d92f0: bl ffff8000100a3110 : mutex_unlock(&ctx->ring_lock); 0.65 : ffff8000102d92f4: mov x0, x22 0.00 : ffff8000102d92f8: bl ffff800010cd4428 : aio_read_events(): : : if (ret > 0) 2.25 : ffff8000102d92fc: cmp x27, #0x0 : *i += ret; 0.00 : ffff8000102d9300: ldr x0, [x20] : if (ret > 0) 0.00 : ffff8000102d9304: b.le ffff8000102d9310 : *i += ret; 0.00 : ffff8000102d9308: add x0, x27, x0 0.00 : ffff8000102d930c: str x0, [x20] : __read_once_size(): : __READ_ONCE_SIZE; 0.32 : ffff8000102d9310: ldr w1, [x28, #56] : aio_read_events(): : : if (unlikely(atomic_read(&ctx->dead))) 0.00 : ffff8000102d9314: cbnz w1, ffff8000102d9454 : ret = -EINVAL; : : if (!*i) 0.00 : ffff8000102d9318: cbz x0, ffff8000102d943c : *i = ret; : : return ret < 0 || *i >= min_nr; 0.00 : ffff8000102d931c: tbnz x27, #63, ffff8000102d9448 0.00 : ffff8000102d9320: ldr x0, [x20] 0.65 : ffff8000102d9324: ldr x27, [x29, #80] 0.00 : ffff8000102d9328: ldr x1, [x29, #104] 0.00 : ffff8000102d932c: cmp x1, x0 0.00 : ffff8000102d9330: cset w0, le : } 0.00 : ffff8000102d9334: ldp x19, x20, [sp, #16] 0.00 : ffff8000102d9338: ldp x21, x22, [sp, #32] 0.32 : ffff8000102d933c: ldp x23, x24, [sp, #48] 0.00 : ffff8000102d9340: ldr x26, [sp, #72] 0.00 : ffff8000102d9344: ldr x28, [sp, #88] 0.00 : ffff8000102d9348: ldp x29, x30, [sp], #128 0.00 : ffff8000102d934c: ret : aio_read_events_ring(): : page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]; 0.00 : ffff8000102d9350: ldr x0, [x28, #168] 0.00 : ffff8000102d9354: lsr w1, w1, #7 0.61 : ffff8000102d9358: str w6, [x29, #116] 0.97 : ffff8000102d935c: str x5, [x29, #120] 0.00 : ffff8000102d9360: ldr x1, [x0, x1, lsl #3] : uaccess_enable_not_uao(): : __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); : } : : static inline void uaccess_enable_not_uao(void) : { : __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102d9364: nop : sign_extend64(): 0.00 : ffff8000102d9368: sbfx x3, x4, #0, #56 : get_current(): 0.00 : ffff8000102d936c: mrs x0, sp_el0 : __uaccess_mask_ptr(): : asm volatile( : " bics xzr, %3, %2\n" : " csel %0, %1, xzr, eq\n" : : "=&r" (safe_ptr) : : "r" (ptr), "r" (current_thread_info()->addr_limit), : "r" (untagged_addr(ptr)) 0.00 : ffff8000102d9370: and x3, x4, x3 : asm volatile( 0.00 : ffff8000102d9374: ldr x11, [x0, #8] 0.00 : ffff8000102d9378: bics xzr, x3, x11 0.00 : ffff8000102d937c: csel x0, x4, xzr, eq // eq = none : : "cc"); : : csdb(); 0.00 : ffff8000102d9380: csdb : lowmem_page_address(): 0.00 : ffff8000102d9384: mov x3, #0x200000 // #2097152 0.00 : ffff8000102d9388: movk x3, #0x200, lsl #32 0.00 : ffff8000102d938c: add x1, x1, x3 0.00 : ffff8000102d9390: mov x3, #0xffff000000000000 // #-281474976710656 0.30 : ffff8000102d9394: lsr x1, x1, #6 0.00 : ffff8000102d9398: add x1, x3, x1, lsl #12 : _copy_to_user(): : n = raw_copy_to_user(to, from, n); 0.00 : ffff8000102d939c: add x1, x1, x10, lsl #5 0.00 : ffff8000102d93a0: bl ffff800010cb7700 <__arch_copy_to_user> 0.00 : ffff8000102d93a4: mov x2, x0 : uaccess_disable_not_uao(): : __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102d93a8: nop 0.29 : ffff8000102d93ac: ldr w6, [x29, #116] 0.65 : ffff8000102d93b0: ldr x5, [x29, #120] : aio_read_events_ring(): : if (unlikely(copy_ret)) { 0.00 : ffff8000102d93b4: cbz w2, ffff8000102d9268 : mutex_unlock(&ctx->ring_lock); 0.00 : ffff8000102d93b8: mov x0, x22 0.00 : ffff8000102d93bc: bl ffff800010cd4428 : __read_once_size(): 0.00 : ffff8000102d93c0: ldr w0, [x28, #56] : aio_read_events(): : if (unlikely(atomic_read(&ctx->dead))) 0.00 : ffff8000102d93c4: cbnz w0, ffff8000102d9474 : if (!*i) 0.00 : ffff8000102d93c8: ldr x0, [x20] 0.00 : ffff8000102d93cc: cbnz x0, ffff8000102d9444 : *i = ret; 0.00 : ffff8000102d93d0: mov x0, #0xfffffffffffffff2 // #-14 0.00 : ffff8000102d93d4: str x0, [x20] 0.00 : ffff8000102d93d8: ldr x25, [x29, #64] 0.00 : ffff8000102d93dc: ldr x27, [x29, #80] 0.00 : ffff8000102d93e0: mov w0, #0x1 // #1 : } 0.00 : ffff8000102d93e4: ldp x19, x20, [sp, #16] 0.00 : ffff8000102d93e8: ldp x21, x22, [sp, #32] 0.00 : ffff8000102d93ec: ldp x23, x24, [sp, #48] 0.00 : ffff8000102d93f0: ldr x26, [sp, #72] 0.00 : ffff8000102d93f4: ldr x28, [sp, #88] 0.00 : ffff8000102d93f8: ldp x29, x30, [sp], #128 0.00 : ffff8000102d93fc: ret : __kunmap_atomic(): 0.33 : ffff8000102d9400: bl ffff800010cd2e50 : aio_read_events_ring(): : smp_rmb(); 0.00 : ffff8000102d9404: dmb ishld : if (head == tail) 0.00 : ffff8000102d9408: cmp w26, w24 0.00 : ffff8000102d940c: b.ne ffff8000102d91a8 // b.any : mutex_unlock(&ctx->ring_lock); 0.00 : ffff8000102d9410: mov x0, x22 0.00 : ffff8000102d9414: bl ffff800010cd4428 : __read_once_size(): 0.30 : ffff8000102d9418: ldr w0, [x28, #56] : aio_read_events(): : if (unlikely(atomic_read(&ctx->dead))) 0.00 : ffff8000102d941c: cbnz w0, ffff8000102d9484 : if (!*i) 0.00 : ffff8000102d9420: ldr x0, [x20] 0.00 : ffff8000102d9424: b ffff8000102d9328 : check_copy_size(): 0.00 : ffff8000102d9428: brk #0x800 0.00 : ffff8000102d942c: b ffff8000102d9264 : __read_once_size(): 3.23 : ffff8000102d9430: ldr x0, [x0, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102d9434: cbz x0, ffff8000102d92e4 7.67 : ffff8000102d9438: b ffff8000102d92e8 : aio_read_events(): : *i = ret; 0.00 : ffff8000102d943c: str x27, [x20] 0.00 : ffff8000102d9440: b ffff8000102d931c 0.00 : ffff8000102d9444: ldr x25, [x29, #64] 0.00 : ffff8000102d9448: ldr x27, [x29, #80] 0.00 : ffff8000102d944c: mov w0, #0x1 // #1 0.00 : ffff8000102d9450: b ffff8000102d93e4 0.00 : ffff8000102d9454: ldr x27, [x29, #80] : if (!*i) 0.00 : ffff8000102d9458: cbnz x0, ffff8000102d93e0 : *i = ret; 0.00 : ffff8000102d945c: mov x1, #0xffffffffffffffea // #-22 0.00 : ffff8000102d9460: mov w0, #0x1 // #1 0.00 : ffff8000102d9464: str x1, [x20] : return ret < 0 || *i >= min_nr; 0.00 : ffff8000102d9468: b ffff8000102d9334 : aio_read_events_ring(): : long ret = 0; 0.00 : ffff8000102d946c: mov x27, #0x0 // #0 0.00 : ffff8000102d9470: b ffff8000102d9288 0.00 : ffff8000102d9474: ldr x0, [x20] 0.00 : ffff8000102d9478: ldr x25, [x29, #64] 0.00 : ffff8000102d947c: ldr x27, [x29, #80] 0.00 : ffff8000102d9480: b ffff8000102d9458 0.00 : ffff8000102d9484: ldr x0, [x20] 0.00 : ffff8000102d9488: b ffff8000102d9458 Percent | Source code & Disassembly of vmlinux for cycles (102 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010127620 : : newidle_balance(): : * < 0 - we released the lock and there are !fair tasks present : * 0 - failed, no new tasks : * > 0 - success, new (fair) tasks present : */ : int newidle_balance(struct rq *this_rq, struct rq_flags *rf) : { 3.87 : ffff800010127620: stp x29, x30, [sp, #-128]! 0.00 : ffff800010127624: mov x29, sp 3.03 : ffff800010127628: stp x19, x20, [sp, #16] 0.00 : ffff80001012762c: mov x19, x0 0.00 : ffff800010127630: stp x21, x22, [sp, #32] : unsigned long next_balance = jiffies + HZ; 0.00 : ffff800010127634: adrp x20, ffff800011907000 : { 2.90 : ffff800010127638: str x23, [sp, #48] 0.00 : ffff80001012763c: adrp x23, ffff800011909000 0.93 : ffff800010127640: str x26, [sp, #72] 0.00 : ffff800010127644: add x1, x23, #0x908 0.95 : ffff800010127648: str x28, [sp, #88] 0.00 : ffff80001012764c: ldr x0, [x1] 0.00 : ffff800010127650: str x0, [x29, #120] 0.00 : ffff800010127654: mov x0, #0x0 // #0 : int this_cpu = this_rq->cpu; 1.99 : ffff800010127658: ldr w21, [x19, #2568] : unsigned long next_balance = jiffies + HZ; 0.93 : ffff80001012765c: ldr x22, [x20, #2432] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010127660: nop : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010127664: add w0, w21, #0x3f 0.00 : ffff800010127668: cmp w21, #0x0 0.00 : ffff80001012766c: csel w0, w0, w21, lt // lt = tstop 0.00 : ffff800010127670: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : newidle_balance(): : update_misfit_status(NULL, this_rq); : /* : * We must set idle_stamp _before_ calling idle_balance(), such that we : * measure the duration of idle_balance() as idle time. : */ : this_rq->idle_stamp = rq_clock(this_rq); 2.94 : ffff800010127674: ldr x2, [x19, #2400] : test_bit(): 0.00 : ffff800010127678: add x1, x1, #0x1c8 0.00 : ffff80001012767c: asr w0, w0, #6 : newidle_balance(): 4.94 : ffff800010127680: str x2, [x19, #2816] : test_bit(): 0.00 : ffff800010127684: sxtw x0, w0 0.00 : ffff800010127688: ldr x0, [x1, x0, lsl #3] 0.00 : ffff80001012768c: lsr x0, x0, x21 : newidle_balance(): : : /* : * Do not pull tasks towards !active CPUs... : */ : if (!cpu_active(this_cpu)) 0.00 : ffff800010127690: and w26, w0, #0x1 0.00 : ffff800010127694: tbz w0, #0, ffff800010127798 : * further scheduler activity on it and we're being very careful to : * re-start the picking loop. : */ : rq_unpin_lock(this_rq, rf); : : if (this_rq->avg_idle < sysctl_sched_migration_cost || 5.00 : ffff800010127698: ldr x1, [x19, #2824] 0.00 : ffff80001012769c: mov x0, #0xa11f // #41247 0.00 : ffff8000101276a0: movk x0, #0x7, lsl #16 : unsigned long next_balance = jiffies + HZ; 0.00 : ffff8000101276a4: add x22, x22, #0xfa : if (this_rq->avg_idle < sysctl_sched_migration_cost || 4.99 : ffff8000101276a8: cmp x1, x0 0.00 : ffff8000101276ac: b.ls ffff8000101276bc // b.plast : !READ_ONCE(this_rq->rd->overload)) { 0.00 : ffff8000101276b0: ldr x0, [x19, #2464] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101276b4: ldr w0, [x0, #88] : newidle_balance(): : if (this_rq->avg_idle < sysctl_sched_migration_cost || 0.00 : ffff8000101276b8: cbnz w0, ffff8000101277f0 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 3.99 : ffff8000101276bc: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 1.02 : ffff8000101276c0: ldr x21, [x19, #2472] : newidle_balance(): : : rcu_read_lock(); : sd = rcu_dereference_check_sched_domain(this_rq->sd); : if (sd) 0.00 : ffff8000101276c4: cbz x21, ffff8000101276fc : msecs_to_jiffies(): : if (__builtin_constant_p(m)) { : if ((int)m < 0) : return MAX_JIFFY_OFFSET; : return _msecs_to_jiffies(m); : } else { : return __msecs_to_jiffies(m); 0.92 : ffff8000101276c8: ldr w0, [x21, #72] 0.00 : ffff8000101276cc: bl ffff800010167188 <__msecs_to_jiffies> : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 1.96 : ffff8000101276d0: cmp x0, #0x0 0.00 : ffff8000101276d4: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101276d8: csinc x0, x0, xzr, ne // ne = any : update_next_balance(): : next = sd->last_balance + interval; 0.94 : ffff8000101276dc: ldr x2, [x21, #64] : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 2.87 : ffff8000101276e0: ldr x1, [x1, #584] 0.00 : ffff8000101276e4: cmp x0, x1 0.00 : ffff8000101276e8: csel x0, x0, x1, ls // ls = plast : update_next_balance(): : next = sd->last_balance + interval; 0.00 : ffff8000101276ec: add x0, x0, x2 : if (time_after(*next_balance, next)) 1.87 : ffff8000101276f0: sub x1, x0, x22 : *next_balance = next; 0.00 : ffff8000101276f4: cmp x1, #0x0 0.00 : ffff8000101276f8: csel x22, x22, x0, ge // ge = tcont : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff8000101276fc: bl ffff800010160d20 <__rcu_read_unlock> : arch_static_branch(): 0.89 : ffff800010127700: nop : nohz_newidle_balance(): : if (this_rq->avg_idle < sysctl_sched_migration_cost) 0.94 : ffff800010127704: ldr x1, [x19, #2824] 0.00 : ffff800010127708: mov x0, #0xa11f // #41247 0.00 : ffff80001012770c: movk x0, #0x7, lsl #16 0.00 : ffff800010127710: cmp x1, x0 2.04 : ffff800010127714: b.ls ffff800010127760 // b.plast : __read_once_size(): 0.00 : ffff800010127718: adrp x0, ffff800011afb000 0.00 : ffff80001012771c: add x0, x0, #0xfc0 0.00 : ffff800010127720: ldr w1, [x0, #36] : nohz_newidle_balance(): : if (!READ_ONCE(nohz.has_blocked) || 0.00 : ffff800010127724: cbz w1, ffff800010127760 : time_before(jiffies, READ_ONCE(nohz.next_blocked))) 0.00 : ffff800010127728: ldr x1, [x20, #2432] : __read_once_size(): 0.00 : ffff80001012772c: ldr x0, [x0, #48] : nohz_newidle_balance(): 0.00 : ffff800010127730: sub x0, x1, x0 0.00 : ffff800010127734: tbnz x0, #63, ffff800010127760 : raw_spin_unlock(&this_rq->lock); 0.00 : ffff800010127738: mov x0, x19 0.00 : ffff80001012773c: bl ffff800010cd7c38 <_raw_spin_unlock> : if (!_nohz_idle_balance(this_rq, NOHZ_STATS_KICK, CPU_NEWLY_IDLE)) 0.00 : ffff800010127740: mov w2, #0x2 // #2 0.00 : ffff800010127744: mov x0, x19 0.00 : ffff800010127748: mov w1, w2 0.00 : ffff80001012774c: bl ffff800010127130 <_nohz_idle_balance> 0.00 : ffff800010127750: tst w0, #0xff 0.00 : ffff800010127754: b.eq ffff8000101279b0 // b.none : newidle_balance(): : if (pulled_task || this_rq->nr_running > 0) : break; : } : rcu_read_unlock(); : : raw_spin_lock(&this_rq->lock); 0.00 : ffff800010127758: mov x0, x19 0.00 : ffff80001012775c: bl ffff800010cd80f0 <_raw_spin_lock> : /* : * While browsing the domains, we released the rq lock, a task could : * have been enqueued in the meantime. Since we're not going idle, : * pretend we pulled a task. : */ : if (this_rq->cfs.h_nr_running && !pulled_task) 1.99 : ffff800010127760: ldr w2, [x19, #156] 0.00 : ffff800010127764: cmp w2, #0x0 0.00 : ffff800010127768: cset w28, ne // ne = any : pulled_task = 1; : : /* Move the next balance forward */ : if (time_after(this_rq->next_balance, next_balance)) 3.05 : ffff80001012776c: ldr x0, [x19, #2376] 0.00 : ffff800010127770: sub x0, x22, x0 0.00 : ffff800010127774: tbz x0, #63, ffff80001012777c : this_rq->next_balance = next_balance; 0.00 : ffff800010127778: str x22, [x19, #2376] : : /* Is there a task of a high priority class? */ : if (this_rq->nr_running != this_rq->cfs.h_nr_running) 0.00 : ffff80001012777c: ldr w0, [x19, #4] 25.32 : ffff800010127780: cmp w2, w0 0.00 : ffff800010127784: b.eq ffff800010127794 // b.none : pulled_task = -1; 0.00 : ffff800010127788: mov w28, #0xffffffff // #-1 : : if (pulled_task) : this_rq->idle_stamp = 0; 0.00 : ffff80001012778c: str xzr, [x19, #2816] 0.00 : ffff800010127790: b ffff80001012779c : if (pulled_task) 0.00 : ffff800010127794: cbnz w28, ffff80001012778c : return 0; 0.00 : ffff800010127798: mov w28, #0x0 // #0 : : rq_repin_lock(this_rq, rf); : : return pulled_task; : } 0.00 : ffff80001012779c: add x23, x23, #0x908 0.00 : ffff8000101277a0: mov w0, w28 0.97 : ffff8000101277a4: ldr x2, [x29, #120] 2.95 : ffff8000101277a8: ldr x1, [x23] 0.00 : ffff8000101277ac: eor x1, x2, x1 0.00 : ffff8000101277b0: cbnz x1, ffff8000101279c4 1.98 : ffff8000101277b4: ldp x19, x20, [sp, #16] 5.89 : ffff8000101277b8: ldp x21, x22, [sp, #32] 1.99 : ffff8000101277bc: ldr x23, [sp, #48] 0.00 : ffff8000101277c0: ldr x26, [sp, #72] 4.97 : ffff8000101277c4: ldr x28, [sp, #88] 0.98 : ffff8000101277c8: ldp x29, x30, [sp], #128 0.00 : ffff8000101277cc: ret : update_misfit_status(): : rq->misfit_task_load = 0; 0.00 : ffff8000101277d0: str xzr, [x19, #2512] 0.00 : ffff8000101277d4: b ffff800010127664 : housekeeping_cpu(): : : static inline bool housekeeping_cpu(int cpu, enum hk_flags flags) : { : #ifdef CONFIG_CPU_ISOLATION : if (static_branch_unlikely(&housekeeping_overridden)) : return housekeeping_test_cpu(cpu, flags); 0.00 : ffff8000101277d8: ldr w0, [x19, #2568] 0.00 : ffff8000101277dc: mov w1, #0x8 // #8 0.00 : ffff8000101277e0: bl ffff800010137e78 : nohz_newidle_balance(): : if (!housekeeping_cpu(this_cpu, HK_FLAG_SCHED)) 0.00 : ffff8000101277e4: tst w0, #0xff 0.00 : ffff8000101277e8: b.eq ffff800010127760 // b.none 0.00 : ffff8000101277ec: b ffff800010127704 : newidle_balance(): : raw_spin_unlock(&this_rq->lock); 0.00 : ffff8000101277f0: mov x0, x19 0.00 : ffff8000101277f4: bl ffff800010cd7c38 <_raw_spin_unlock> : update_blocked_averages(this_cpu); 0.00 : ffff8000101277f8: mov w0, w21 0.00 : ffff8000101277fc: bl ffff8000101226f8 : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff800010127800: bl ffff80001015c1b0 <__rcu_read_lock> : newidle_balance(): : for_each_domain(this_cpu, sd) { 0.00 : ffff800010127804: adrp x1, ffff800011909000 0.00 : ffff800010127808: add x1, x1, #0x928 0.00 : ffff80001012780c: adrp x0, ffff800011528000 0.00 : ffff800010127810: add x0, x0, #0xe80 0.00 : ffff800010127814: ldr x1, [x1, w21, sxtw #3] 0.00 : ffff800010127818: add x0, x0, x1 : __read_once_size(): 0.00 : ffff80001012781c: ldr x20, [x0, #2472] : newidle_balance(): 0.00 : ffff800010127820: cbz x20, ffff8000101279bc 0.00 : ffff800010127824: stp x24, x25, [x29, #56] : u64 curr_cost = 0; 0.00 : ffff800010127828: mov x24, #0x0 // #0 0.00 : ffff80001012782c: str x27, [x29, #80] : int continue_balancing = 1; 0.00 : ffff800010127830: mov w25, #0x1 // #1 : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff800010127834: adrp x27, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010127838: add x27, x27, #0x240 0.00 : ffff80001012783c: b ffff800010127880 : msecs_to_jiffies(): 0.00 : ffff800010127840: ldr w0, [x20, #72] 0.00 : ffff800010127844: bl ffff800010167188 <__msecs_to_jiffies> : get_sd_balance_interval(): 0.00 : ffff800010127848: cmp x0, #0x0 0.00 : ffff80001012784c: ldr x1, [x27, #8] 0.00 : ffff800010127850: csinc x0, x0, xzr, ne // ne = any : update_next_balance(): : next = sd->last_balance + interval; 0.00 : ffff800010127854: ldr x2, [x20, #64] : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff800010127858: cmp x0, x1 0.00 : ffff80001012785c: csel x0, x0, x1, ls // ls = plast : update_next_balance(): : next = sd->last_balance + interval; 0.00 : ffff800010127860: add x0, x0, x2 : if (time_after(*next_balance, next)) 0.00 : ffff800010127864: sub x1, x0, x22 : next = sd->last_balance + interval; 0.00 : ffff800010127868: cmp x1, #0x0 0.00 : ffff80001012786c: csel x22, x22, x0, ge // ge = tcont : newidle_balance(): : if (pulled_task || this_rq->nr_running > 0) 0.00 : ffff800010127870: ldr w0, [x19, #4] 0.00 : ffff800010127874: cbnz w0, ffff800010127960 : for_each_domain(this_cpu, sd) { 0.00 : ffff800010127878: ldr x20, [x20] 0.00 : ffff80001012787c: cbz x20, ffff800010127960 : if (!(sd->flags & SD_LOAD_BALANCE)) 0.00 : ffff800010127880: ldr w1, [x20, #56] : int continue_balancing = 1; 0.00 : ffff800010127884: str w25, [x29, #116] : if (!(sd->flags & SD_LOAD_BALANCE)) 0.00 : ffff800010127888: tbz w1, #0, ffff800010127878 : if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { 0.00 : ffff80001012788c: ldr x0, [x20, #80] 0.00 : ffff800010127890: ldr x2, [x19, #2824] 0.00 : ffff800010127894: add x0, x24, x0 0.00 : ffff800010127898: cmp x2, x0 0.00 : ffff80001012789c: b.cc ffff800010127974 // b.lo, b.ul, b.last : if (sd->flags & SD_BALANCE_NEWIDLE) { 0.00 : ffff8000101278a0: tbz w1, #1, ffff800010127840 : t0 = sched_clock_cpu(this_cpu); 0.00 : ffff8000101278a4: mov w0, w21 0.00 : ffff8000101278a8: bl ffff80001011b140 0.00 : ffff8000101278ac: mov x5, x0 : pulled_task = load_balance(this_cpu, this_rq, 0.00 : ffff8000101278b0: add x4, x29, #0x74 0.00 : ffff8000101278b4: mov x2, x20 0.00 : ffff8000101278b8: mov x1, x19 0.00 : ffff8000101278bc: mov w3, #0x2 // #2 : t0 = sched_clock_cpu(this_cpu); 0.00 : ffff8000101278c0: str x5, [x29, #104] : pulled_task = load_balance(this_cpu, this_rq, 0.00 : ffff8000101278c4: mov w0, w21 0.00 : ffff8000101278c8: bl ffff8000101261a8 0.00 : ffff8000101278cc: mov w28, w0 : domain_cost = sched_clock_cpu(this_cpu) - t0; 0.00 : ffff8000101278d0: mov w0, w21 0.00 : ffff8000101278d4: bl ffff80001011b140 0.00 : ffff8000101278d8: ldr x5, [x29, #104] : if (domain_cost > sd->max_newidle_lb_cost) 0.00 : ffff8000101278dc: ldr x3, [x20, #80] : domain_cost = sched_clock_cpu(this_cpu) - t0; 0.00 : ffff8000101278e0: sub x5, x0, x5 : if (domain_cost > sd->max_newidle_lb_cost) 0.00 : ffff8000101278e4: cmp x3, x5 0.00 : ffff8000101278e8: b.cs ffff8000101278f0 // b.hs, b.nlast : sd->max_newidle_lb_cost = domain_cost; 0.00 : ffff8000101278ec: str x5, [x20, #80] : msecs_to_jiffies(): 0.00 : ffff8000101278f0: ldr w0, [x20, #72] : newidle_balance(): : curr_cost += domain_cost; 0.00 : ffff8000101278f4: add x24, x24, x5 : msecs_to_jiffies(): 0.00 : ffff8000101278f8: bl ffff800010167188 <__msecs_to_jiffies> : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff8000101278fc: cmp x0, #0x0 0.00 : ffff800010127900: ldr x2, [x27, #8] 0.00 : ffff800010127904: csinc x0, x0, xzr, ne // ne = any : update_next_balance(): : next = sd->last_balance + interval; 0.00 : ffff800010127908: ldr x3, [x20, #64] : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff80001012790c: cmp x0, x2 0.00 : ffff800010127910: csel x0, x0, x2, ls // ls = plast : update_next_balance(): : next = sd->last_balance + interval; 0.00 : ffff800010127914: add x0, x0, x3 : if (time_after(*next_balance, next)) 0.00 : ffff800010127918: sub x2, x0, x22 : next = sd->last_balance + interval; 0.00 : ffff80001012791c: cmp x2, #0x0 0.00 : ffff800010127920: csel x22, x22, x0, ge // ge = tcont : newidle_balance(): : if (pulled_task || this_rq->nr_running > 0) 0.00 : ffff800010127924: cbz w28, ffff800010127870 : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff800010127928: bl ffff800010160d20 <__rcu_read_unlock> : newidle_balance(): : raw_spin_lock(&this_rq->lock); 0.00 : ffff80001012792c: mov x0, x19 0.00 : ffff800010127930: bl ffff800010cd80f0 <_raw_spin_lock> : if (curr_cost > this_rq->max_idle_balance_cost) 0.00 : ffff800010127934: ldr x0, [x19, #2832] 0.00 : ffff800010127938: cmp x0, x24 0.00 : ffff80001012793c: b.cs ffff800010127944 // b.hs, b.nlast : this_rq->max_idle_balance_cost = curr_cost; 0.00 : ffff800010127940: str x24, [x19, #2832] : if (this_rq->cfs.h_nr_running && !pulled_task) 0.00 : ffff800010127944: ldr w2, [x19, #156] 0.00 : ffff800010127948: cbz w2, ffff800010127968 : pulled_task = 1; 0.00 : ffff80001012794c: cmp w28, #0x0 0.00 : ffff800010127950: ldr x27, [x29, #80] 0.00 : ffff800010127954: csel w28, w28, w26, ne // ne = any 0.00 : ffff800010127958: ldp x24, x25, [x29, #56] 0.00 : ffff80001012795c: b ffff80001012776c 0.00 : ffff800010127960: mov w28, #0x0 // #0 0.00 : ffff800010127964: b ffff800010127928 0.00 : ffff800010127968: ldp x24, x25, [x29, #56] 0.00 : ffff80001012796c: ldr x27, [x29, #80] 0.00 : ffff800010127970: b ffff80001012776c : msecs_to_jiffies(): 0.00 : ffff800010127974: ldr w0, [x20, #72] : update_next_balance(): : *next_balance = next; 0.00 : ffff800010127978: mov w28, #0x0 // #0 : msecs_to_jiffies(): 0.00 : ffff80001012797c: bl ffff800010167188 <__msecs_to_jiffies> : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff800010127980: cmp x0, #0x0 0.00 : ffff800010127984: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010127988: csinc x0, x0, xzr, ne // ne = any : update_next_balance(): : next = sd->last_balance + interval; 0.00 : ffff80001012798c: ldr x3, [x20, #64] : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff800010127990: ldr x2, [x2, #584] 0.00 : ffff800010127994: cmp x0, x2 0.00 : ffff800010127998: csel x0, x0, x2, ls // ls = plast : update_next_balance(): : next = sd->last_balance + interval; 0.00 : ffff80001012799c: add x0, x0, x3 : if (time_after(*next_balance, next)) 0.00 : ffff8000101279a0: sub x2, x0, x22 : *next_balance = next; 0.00 : ffff8000101279a4: cmp x2, #0x0 0.00 : ffff8000101279a8: csel x22, x22, x0, ge // ge = tcont 0.00 : ffff8000101279ac: b ffff800010127928 : nohz_newidle_balance(): : kick_ilb(NOHZ_STATS_KICK); 0.00 : ffff8000101279b0: mov w0, #0x2 // #2 0.00 : ffff8000101279b4: bl ffff80001011e090 0.00 : ffff8000101279b8: b ffff800010127758 : rcu_read_unlock(): 0.00 : ffff8000101279bc: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff8000101279c0: b ffff800010127758 0.00 : ffff8000101279c4: stp x24, x25, [x29, #56] 0.00 : ffff8000101279c8: str x27, [x29, #80] : newidle_balance(): : } 0.00 : ffff8000101279cc: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (100 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016aeb8 : : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001016aeb8: ldr x1, [x0, #48] 2.01 : ffff80001016aebc: ldr w2, [x1, #16] : __read_seqcount_begin(): : { : unsigned ret; : : repeat: : ret = READ_ONCE(s->sequence); : if (unlikely(ret & 1)) { 1.04 : ffff80001016aec0: tbnz w2, #0, ffff80001016af00 : raw_read_seqcount_begin(): : * section is tested by checking read_seqcount_retry function. : */ : static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) : { : unsigned ret = __read_seqcount_begin(s); : smp_rmb(); 5.87 : ffff80001016aec4: dmb ishld : hrtimer_active(): : : do { : base = READ_ONCE(timer->base); : seq = raw_read_seqcount_begin(&base->seq); : : if (timer->state != HRTIMER_STATE_INACTIVE || 91.08 : ffff80001016aec8: ldrb w3, [x0, #56] 0.00 : ffff80001016aecc: cbnz w3, ffff80001016af10 0.00 : ffff80001016aed0: ldr x3, [x1, #24] 0.00 : ffff80001016aed4: cmp x3, x0 0.00 : ffff80001016aed8: b.eq ffff80001016af10 // b.none : read_seqcount_retry(): : * If the critical section was invalid, it must be ignored (and typically : * retried). : */ : static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) : { : smp_rmb(); 0.00 : ffff80001016aedc: dmb ishld : hrtimer_active(): : base->running == timer) : return true; : : } while (read_seqcount_retry(&base->seq, seq) || : base != READ_ONCE(timer->base)); 0.00 : ffff80001016aee0: ldr w3, [x1, #16] 0.00 : ffff80001016aee4: cmp w3, w2 0.00 : ffff80001016aee8: b.ne ffff80001016aeb8 // b.any : __read_once_size(): 0.00 : ffff80001016aeec: ldr x2, [x0, #48] : hrtimer_active(): : } while (read_seqcount_retry(&base->seq, seq) || 0.00 : ffff80001016aef0: cmp x1, x2 0.00 : ffff80001016aef4: b.ne ffff80001016aeb8 // b.any : : return false; 0.00 : ffff80001016aef8: mov w0, #0x0 // #0 : } 0.00 : ffff80001016aefc: ret : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff80001016af00: yield : __read_once_size(): 0.00 : ffff80001016af04: ldr w2, [x1, #16] : __read_seqcount_begin(): : if (unlikely(ret & 1)) { 0.00 : ffff80001016af08: tbz w2, #0, ffff80001016aec4 0.00 : ffff80001016af0c: b ffff80001016af00 : hrtimer_active(): : return true; 0.00 : ffff80001016af10: mov w0, #0x1 // #1 : } 0.00 : ffff80001016af14: ret Percent | Source code & Disassembly of vmlinux for cycles (291 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d95e8 : : lookup_ioctx(): : req->ki_eventfd = NULL; : return req; : } : : static struct kioctx *lookup_ioctx(unsigned long ctx_id) : { 0.35 : ffff8000102d95e8: stp x29, x30, [sp, #-48]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000102d95ec: mrs x1, sp_el0 : lookup_ioctx(): 0.00 : ffff8000102d95f0: mov x29, sp 0.00 : ffff8000102d95f4: str x20, [sp, #24] 2.10 : ffff8000102d95f8: str x22, [sp, #40] : __range_ok(): : * Asynchronous I/O running in a kernel thread does not have the : * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag : * the user address before checking. : */ : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && : (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) 0.70 : ffff8000102d95fc: ldr w3, [x1, #44] : unsigned long ret, limit = current_thread_info()->addr_limit; 2.42 : ffff8000102d9600: ldr x2, [x1, #8] : lookup_ioctx(): : struct aio_ring __user *ring = (void __user *)ctx_id; : struct mm_struct *mm = current->mm; 0.00 : ffff8000102d9604: ldr x22, [x1, #952] : __range_ok(): : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 0.00 : ffff8000102d9608: tbnz w3, #21, ffff8000102d961c : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 1.32 : ffff8000102d960c: ldr x3, [x1] : __range_ok(): 0.00 : ffff8000102d9610: mov x1, x0 0.00 : ffff8000102d9614: tst w3, #0x4000000 0.00 : ffff8000102d9618: b.eq ffff8000102d9624 // b.none : sign_extend64(): : * @index: 0 based bit index (0<=index<64) to sign bit : */ : static inline __s64 sign_extend64(__u64 value, int index) : { : __u8 shift = 63 - index; : return (__s64)(value << shift) >> shift; 0.00 : ffff8000102d961c: sbfx x1, x0, #0, #56 : __range_ok(): : addr = untagged_addr(addr); 0.00 : ffff8000102d9620: and x1, x0, x1 : : __chk_user_ptr(addr); : asm volatile( 0.35 : ffff8000102d9624: adds x1, x1, #0x4 0.00 : ffff8000102d9628: csel x2, xzr, x2, hi // hi = pmore 0.00 : ffff8000102d962c: csinv x1, x1, xzr, cc // cc = lo, ul, last 0.00 : ffff8000102d9630: sbcs xzr, x1, x2 0.00 : ffff8000102d9634: cset x1, ls // ls = plast : lookup_ioctx(): : struct kioctx *ctx, *ret = NULL; : struct kioctx_table *table; : unsigned id; : : if (get_user(id, &ring->id)) 0.00 : ffff8000102d9638: cbz x1, ffff8000102d96ec 0.00 : ffff8000102d963c: str x21, [x29, #32] : sign_extend64(): 0.00 : ffff8000102d9640: sbfx x1, x0, #0, #56 : get_current(): 1.74 : ffff8000102d9644: mrs x21, sp_el0 : __uaccess_mask_ptr(): : asm volatile( : " bics xzr, %3, %2\n" : " csel %0, %1, xzr, eq\n" : : "=&r" (safe_ptr) : : "r" (ptr), "r" (current_thread_info()->addr_limit), : "r" (untagged_addr(ptr)) 0.00 : ffff8000102d9648: and x1, x0, x1 : asm volatile( 0.00 : ffff8000102d964c: ldr x3, [x21, #8] 0.00 : ffff8000102d9650: bics xzr, x1, x3 0.00 : ffff8000102d9654: csel x2, x0, xzr, eq // eq = none : : "cc"); : : csdb(); 0.00 : ffff8000102d9658: csdb : uaccess_enable_not_uao(): : __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102d965c: nop : lookup_ioctx(): 0.31 : ffff8000102d9660: mov w1, #0x0 // #0 0.00 : ffff8000102d9664: ldr w20, [x2] : uaccess_disable_not_uao(): : __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102d9668: nop : lookup_ioctx(): 27.19 : ffff8000102d966c: cbnz w1, ffff8000102d96e8 0.00 : ffff8000102d9670: str x19, [x29, #16] 0.00 : ffff8000102d9674: mov x19, x0 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000102d9678: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000102d967c: ldr x0, [x22, #784] : lookup_ioctx(): : return NULL; : : rcu_read_lock(); : table = rcu_dereference(mm->ioctx_table); : : if (!table || id >= table->nr) 0.00 : ffff8000102d9680: cbz x0, ffff8000102d96c4 0.67 : ffff8000102d9684: ldr w1, [x0, #16] 0.00 : ffff8000102d9688: cmp w1, w20 0.00 : ffff8000102d968c: b.ls ffff8000102d96c4 // b.plast : array_index_mask_nospec(): : static inline unsigned long array_index_mask_nospec(unsigned long idx, : unsigned long sz) : { : unsigned long mask; : : asm volatile( 4.46 : ffff8000102d9690: and x2, x20, #0xffffffff : lookup_ioctx(): : goto out; : : id = array_index_nospec(id, table->nr); 0.00 : ffff8000102d9694: mov w1, w1 : array_index_mask_nospec(): 0.00 : ffff8000102d9698: cmp x2, x1 0.00 : ffff8000102d969c: ngc x1, xzr : " sbc %0, xzr, xzr\n" : : "=r" (mask) : : "r" (idx), "Ir" (sz) : : "cc"); : : csdb(); 1.71 : ffff8000102d96a0: csdb : lookup_ioctx(): : ctx = rcu_dereference(table->table[id]); 0.00 : ffff8000102d96a4: and w20, w20, w1 0.00 : ffff8000102d96a8: add x20, x20, #0x3 : __read_once_size(): 0.00 : ffff8000102d96ac: ldr x22, [x0, x20, lsl #3] : lookup_ioctx(): : if (ctx && ctx->user_id == ctx_id) { 0.00 : ffff8000102d96b0: cbz x22, ffff8000102d96c4 12.81 : ffff8000102d96b4: ldr x0, [x22, #120] : ctx = rcu_dereference(table->table[id]); 0.00 : ffff8000102d96b8: mov x20, x22 : if (ctx && ctx->user_id == ctx_id) { 0.00 : ffff8000102d96bc: cmp x0, x19 0.00 : ffff8000102d96c0: b.eq ffff8000102d9704 // b.none : struct kioctx *ctx, *ret = NULL; 0.00 : ffff8000102d96c4: mov x20, #0x0 // #0 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 1.00 : ffff8000102d96c8: bl ffff800010160d20 <__rcu_read_unlock> : lookup_ioctx(): : if (percpu_ref_tryget_live(&ctx->users)) : ret = ctx; : } : out: : rcu_read_unlock(); : return ret; 0.00 : ffff8000102d96cc: ldr x19, [x29, #16] : } 0.00 : ffff8000102d96d0: mov x0, x20 : return ret; 0.00 : ffff8000102d96d4: ldr x21, [x29, #32] : } 0.00 : ffff8000102d96d8: ldr x20, [sp, #24] 0.69 : ffff8000102d96dc: ldr x22, [sp, #40] 0.35 : ffff8000102d96e0: ldp x29, x30, [sp], #48 0.00 : ffff8000102d96e4: ret 0.00 : ffff8000102d96e8: ldr x21, [x29, #32] : return NULL; 0.00 : ffff8000102d96ec: mov x20, #0x0 // #0 : } 0.00 : ffff8000102d96f0: ldr x22, [sp, #40] 0.00 : ffff8000102d96f4: mov x0, x20 0.00 : ffff8000102d96f8: ldr x20, [sp, #24] 0.00 : ffff8000102d96fc: ldp x29, x30, [sp], #48 0.00 : ffff8000102d9700: ret : rcu_read_lock(): : __rcu_read_lock(); 12.64 : ffff8000102d9704: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.00 : ffff8000102d9708: ldr x0, [x22, #8] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000102d970c: tst x0, #0x3 0.00 : ffff8000102d9710: b.ne ffff8000102d9768 // b.any : __read_once_size(): 2.04 : ffff8000102d9714: ldr w1, [x21, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000102d9718: add w1, w1, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.34 : ffff8000102d971c: str w1, [x21, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102d9720: mov x2, #0x1 // #1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000102d9724: mrs x1, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102d9728: add x0, x0, x1 1.00 : ffff8000102d972c: ldxr x4, [x0] 19.33 : ffff8000102d9730: add x4, x4, x2 0.00 : ffff8000102d9734: stxr w3, x4, [x0] 0.00 : ffff8000102d9738: cbnz w3, ffff8000102d972c : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102d973c: ldr x0, [x21, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102d9740: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 2.01 : ffff8000102d9744: str w0, [x21, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102d9748: cbz x0, ffff8000102d9754 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102d974c: ldr x0, [x21, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102d9750: cbnz x0, ffff8000102d9760 : percpu_ref_tryget_live(): : bool ret = false; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) { : this_cpu_inc(*percpu_count); 0.00 : ffff8000102d9754: bl ffff800010cd2e78 : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff8000102d9758: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff8000102d975c: b ffff8000102d96c8 3.46 : ffff8000102d9760: bl ffff800010160d20 <__rcu_read_unlock> : lookup_ioctx(): 1.00 : ffff8000102d9764: b ffff8000102d96c8 : percpu_ref_tryget_live(): : ret = true; : } else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) { 0.00 : ffff8000102d9768: ldr x0, [x22, #8] 0.00 : ffff8000102d976c: tbz w0, #1, ffff8000102d9778 : rcu_read_unlock(): 0.00 : ffff8000102d9770: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff8000102d9774: b ffff8000102d96c4 : __read_once_size(): 0.00 : ffff8000102d9778: ldr x3, [x22] : atomic64_fetch_add_unless(): : atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) : { : s64 c = atomic64_read(v); : : do { : if (unlikely(c == u)) 0.00 : ffff8000102d977c: cbz x3, ffff8000102d9770 : break; : } while (!atomic64_try_cmpxchg(v, &c, c + a)); 0.00 : ffff8000102d9780: add x2, x3, #0x1 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102d9784: b ffff8000102d97b8 0.00 : ffff8000102d9788: b ffff8000102d97b8 : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff8000102d978c: mov x0, x22 0.00 : ffff8000102d9790: mov x1, x3 0.00 : ffff8000102d9794: mov x4, x1 0.00 : ffff8000102d9798: casal x4, x2, [x22] 0.00 : ffff8000102d979c: mov x0, x4 : atomic64_try_cmpxchg(): : if (unlikely(r != o)) 0.00 : ffff8000102d97a0: cmp x0, x3 0.00 : ffff8000102d97a4: b.eq ffff8000102d9760 // b.none 0.00 : ffff8000102d97a8: mov x3, x0 : atomic64_fetch_add_unless(): : if (unlikely(c == u)) 0.00 : ffff8000102d97ac: cbnz x0, ffff8000102d9780 : rcu_read_unlock(): 0.00 : ffff8000102d97b0: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff8000102d97b4: b ffff8000102d96c4 : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff8000102d97b8: b ffff8000102dd8a4 <__arm64_compat_sys_io_pgetevents_time64+0x234> 0.00 : ffff8000102d97bc: b ffff8000102d97a0 Percent | Source code & Disassembly of vmlinux for cycles (92 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010169768 : : run_timer_softirq(): : : /* : * This function runs timers and the timer-tq in bottom half context. : */ : static __latent_entropy void run_timer_softirq(struct softirq_action *h) : { 31.03 : ffff800010169768: stp x29, x30, [sp, #-160]! 0.00 : ffff80001016976c: mov x29, sp 1.13 : ffff800010169770: str x20, [sp, #24] : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); 0.00 : ffff800010169774: adrp x20, ffff800011523000 : { 0.00 : ffff800010169778: stp x22, x23, [sp, #40] 0.00 : ffff80001016977c: adrp x23, ffff800011909000 0.00 : ffff800010169780: add x0, x23, #0x908 : __run_timers(): : if (!time_after_eq(jiffies, base->clk)) 0.00 : ffff800010169784: adrp x22, ffff800011907000 : run_timer_softirq(): : { 0.00 : ffff800010169788: ldr x1, [x0] 0.00 : ffff80001016978c: str x1, [x29, #152] 0.00 : ffff800010169790: mov x1, #0x0 // #0 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); 0.00 : ffff800010169794: add x20, x20, #0x480 : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff800010169798: mov x1, sp : asm(ALTERNATIVE("mrs %0, tpidr_el1", 1.08 : ffff80001016979c: mrs x0, tpidr_el1 : run_timer_softirq(): 0.00 : ffff8000101697a0: add x20, x20, x0 : __run_timers(): : if (!time_after_eq(jiffies, base->clk)) 0.00 : ffff8000101697a4: ldr x0, [x22, #2432] 0.00 : ffff8000101697a8: ldr x2, [x20, #16] 0.00 : ffff8000101697ac: sub x0, x0, x2 0.00 : ffff8000101697b0: tbz x0, #63, ffff80001016998c : __my_cpu_offset(): 0.00 : ffff8000101697b4: mrs x1, tpidr_el1 : run_timer_softirq(): : : __run_timers(base); : if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) : __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF])); 0.00 : ffff8000101697b8: adrp x20, ffff800011524000 0.00 : ffff8000101697bc: add x20, x20, #0x700 0.00 : ffff8000101697c0: add x20, x20, x1 : __run_timers(): : if (!time_after_eq(jiffies, base->clk)) 0.00 : ffff8000101697c4: ldr x0, [x22, #2432] 0.00 : ffff8000101697c8: ldr x1, [x20, #16] 0.00 : ffff8000101697cc: sub x0, x0, x1 0.00 : ffff8000101697d0: tbnz x0, #63, ffff800010169968 6.70 : ffff8000101697d4: stp x24, x25, [x29, #56] : detach_timer(): : entry->next = LIST_POISON2; 0.00 : ffff8000101697d8: mov x24, #0x122 // #290 : __run_timers(): : raw_spin_lock_irq(&base->lock); 0.00 : ffff8000101697dc: mov x0, x20 2.21 : ffff8000101697e0: str x19, [x29, #16] 0.00 : ffff8000101697e4: str x21, [x29, #32] : detach_timer(): : entry->next = LIST_POISON2; 0.00 : ffff8000101697e8: movk x24, #0xdead, lsl #48 0.00 : ffff8000101697ec: str x26, [x29, #72] : __run_timers(): : raw_spin_lock_irq(&base->lock); 0.00 : ffff8000101697f0: bl ffff800010cd8460 <_raw_spin_lock_irq> : base->must_forward_clk = false; 0.00 : ffff8000101697f4: strb wzr, [x20, #37] : while (time_after_eq(jiffies, base->clk)) { 0.00 : ffff8000101697f8: ldr x1, [x22, #2432] 6.33 : ffff8000101697fc: ldr x0, [x20, #16] 0.00 : ffff800010169800: sub x1, x1, x0 0.00 : ffff800010169804: tbnz x1, #63, ffff800010169950 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 12.07 : ffff800010169808: ldr x19, [x22, #2432] : collect_expired_timers(): : if ((long)(now - base->clk) > 2) { 0.00 : ffff80001016980c: sub x1, x19, x0 0.00 : ffff800010169810: cmp x1, #0x2 0.00 : ffff800010169814: b.le ffff800010169830 : unsigned long next = __next_timer_interrupt(base); 0.00 : ffff800010169818: mov x0, x20 : return 0; 0.00 : ffff80001016981c: mov w25, #0x0 // #0 : unsigned long next = __next_timer_interrupt(base); 0.00 : ffff800010169820: bl ffff8000101691d8 <__next_timer_interrupt> : if (time_after(next, now)) { 0.00 : ffff800010169824: sub x1, x19, x0 0.00 : ffff800010169828: tbnz x1, #63, ffff8000101698ac : base->clk = next; 0.00 : ffff80001016982c: str x0, [x20, #16] 1.10 : ffff800010169830: add x6, x20, #0x28 : __collect_expired_timers(): : int i, levels = 0; 0.00 : ffff800010169834: add x7, x29, #0x50 : vec = base->vectors + idx; 3.12 : ffff800010169838: add x10, x20, #0x70 : collect_expired_timers(): : return 0; 0.00 : ffff80001016983c: mov w3, #0x0 // #0 : __collect_expired_timers(): : int i, levels = 0; 0.00 : ffff800010169840: mov w25, #0x0 // #0 : __test_and_clear_bit(): : * If two examples of this operation race, one can appear to succeed : * but actually fail. You must protect multiple accesses with a lock. : */ : static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) : { : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010169844: mov x9, #0x1 // #1 : __collect_expired_timers(): : idx = (clk & LVL_MASK) + i * LVL_SIZE; 0.00 : ffff800010169848: and w1, w0, #0x3f 0.00 : ffff80001016984c: add w1, w1, w3 : __test_and_clear_bit(): : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff800010169850: asr w2, w1, #6 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010169854: lsl x5, x9, x1 : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff800010169858: sxtw x2, w2 : unsigned long old = *p; 0.00 : ffff80001016985c: ldr x4, [x6, x2, lsl #3] : : *p = old & ~mask; 0.00 : ffff800010169860: bic x8, x4, x5 2.17 : ffff800010169864: str x8, [x6, x2, lsl #3] : __collect_expired_timers(): : if (__test_and_clear_bit(idx, base->pending_map)) { 0.00 : ffff800010169868: tst x5, x4 0.00 : ffff80001016986c: b.eq ffff800010169890 // b.none : hlist_move_list(): : * reference of the first entry if it exists. : */ : static inline void hlist_move_list(struct hlist_head *old, : struct hlist_head *new) : { : new->first = old->first; 0.00 : ffff800010169870: ldr x2, [x10, x1, lsl #3] : __collect_expired_timers(): : hlist_move_list(vec, heads++); 0.00 : ffff800010169874: add x4, x7, #0x8 : hlist_move_list(): 0.00 : ffff800010169878: str x2, [x7] : if (new->first) 0.00 : ffff80001016987c: cbz x2, ffff800010169884 : new->first->pprev = &new->first; 0.00 : ffff800010169880: str x7, [x2, #8] : __collect_expired_timers(): : levels++; 0.00 : ffff800010169884: add w25, w25, #0x1 : hlist_move_list(vec, heads++); 0.00 : ffff800010169888: mov x7, x4 : hlist_move_list(): : old->first = NULL; 0.00 : ffff80001016988c: str xzr, [x10, x1, lsl #3] : __collect_expired_timers(): : if (clk & LVL_CLK_MASK) 0.00 : ffff800010169890: tst x0, #0x7 0.00 : ffff800010169894: b.ne ffff8000101698a8 // b.any 0.00 : ffff800010169898: add w3, w3, #0x40 : clk >>= LVL_CLK_SHIFT; 0.00 : ffff80001016989c: lsr x0, x0, #3 : for (i = 0; i < LVL_DEPTH; i++) { 0.00 : ffff8000101698a0: cmp w3, #0x240 0.00 : ffff8000101698a4: b.ne ffff800010169848 // b.any 0.00 : ffff8000101698a8: ldr x19, [x20, #16] 0.00 : ffff8000101698ac: sub w25, w25, #0x1 0.00 : ffff8000101698b0: add x0, x29, #0x50 : __run_timers(): : base->clk++; 0.00 : ffff8000101698b4: add x19, x19, #0x1 0.00 : ffff8000101698b8: str x19, [x20, #16] 0.00 : ffff8000101698bc: add x21, x0, w25, sxtw #3 : while (levels--) 0.00 : ffff8000101698c0: cmn w25, #0x1 0.00 : ffff8000101698c4: b.eq ffff8000101697f8 // b.none : __read_once_size(): 0.00 : ffff8000101698c8: ldr x0, [x21] : expire_timers(): : while (!hlist_empty(head)) { 0.00 : ffff8000101698cc: cbz x0, ffff800010169920 : timer = hlist_entry(head->first, struct timer_list, entry); 0.00 : ffff8000101698d0: ldr x19, [x21] : base->running_timer = timer; 0.00 : ffff8000101698d4: str x19, [x20, #8] : __hlist_del(): : struct hlist_node **pprev = n->pprev; 0.00 : ffff8000101698d8: ldp x0, x1, [x19] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101698dc: str x0, [x1] : __hlist_del(): : if (next) 0.00 : ffff8000101698e0: cbz x0, ffff8000101698e8 : __write_once_size(): 0.00 : ffff8000101698e4: str x1, [x0, #8] : expire_timers(): : if (timer->flags & TIMER_IRQSAFE) { 0.00 : ffff8000101698e8: ldr w0, [x19, #32] : detach_timer(): : entry->pprev = NULL; 0.00 : ffff8000101698ec: stp x24, xzr, [x19] : expire_timers(): : fn = timer->function; 0.00 : ffff8000101698f0: ldr x26, [x19, #24] : if (timer->flags & TIMER_IRQSAFE) { 0.00 : ffff8000101698f4: tbz w0, #21, ffff80001016992c : raw_spin_unlock(&base->lock); 0.00 : ffff8000101698f8: mov x0, x20 0.00 : ffff8000101698fc: bl ffff800010cd7c38 <_raw_spin_unlock> : call_timer_fn(timer, fn, baseclk); 0.00 : ffff800010169900: mov x1, x26 0.00 : ffff800010169904: mov x0, x19 0.00 : ffff800010169908: bl ffff8000101693a8 : base->running_timer = NULL; 0.00 : ffff80001016990c: str xzr, [x20, #8] : raw_spin_lock(&base->lock); 0.00 : ffff800010169910: mov x0, x20 0.00 : ffff800010169914: bl ffff800010cd80f0 <_raw_spin_lock> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010169918: ldr x0, [x21] : expire_timers(): : while (!hlist_empty(head)) { 0.00 : ffff80001016991c: cbnz x0, ffff8000101698d0 0.00 : ffff800010169920: sub x21, x21, #0x8 0.00 : ffff800010169924: sub w25, w25, #0x1 0.00 : ffff800010169928: b ffff8000101698c0 : raw_spin_unlock_irq(&base->lock); 0.00 : ffff80001016992c: mov x0, x20 0.00 : ffff800010169930: bl ffff800010cd7c70 <_raw_spin_unlock_irq> : call_timer_fn(timer, fn, baseclk); 0.00 : ffff800010169934: mov x1, x26 0.00 : ffff800010169938: mov x0, x19 0.00 : ffff80001016993c: bl ffff8000101693a8 : base->running_timer = NULL; 0.00 : ffff800010169940: str xzr, [x20, #8] : raw_spin_lock_irq(&base->lock); 0.00 : ffff800010169944: mov x0, x20 0.00 : ffff800010169948: bl ffff800010cd8460 <_raw_spin_lock_irq> 0.00 : ffff80001016994c: b ffff8000101698c8 : __run_timers(): : raw_spin_unlock_irq(&base->lock); 0.00 : ffff800010169950: mov x0, x20 0.00 : ffff800010169954: bl ffff800010cd7c70 <_raw_spin_unlock_irq> 0.00 : ffff800010169958: ldr x19, [x29, #16] 1.13 : ffff80001016995c: ldr x21, [x29, #32] 0.00 : ffff800010169960: ldp x24, x25, [x29, #56] 1.09 : ffff800010169964: ldr x26, [x29, #72] : run_timer_softirq(): : } 0.00 : ffff800010169968: add x23, x23, #0x908 0.00 : ffff80001016996c: ldr x1, [x29, #152] 1.11 : ffff800010169970: ldr x0, [x23] 0.00 : ffff800010169974: eor x0, x1, x0 0.00 : ffff800010169978: cbnz x0, ffff800010169b28 0.00 : ffff80001016997c: ldr x20, [sp, #24] 0.00 : ffff800010169980: ldp x22, x23, [sp, #40] 0.00 : ffff800010169984: ldp x29, x30, [sp], #160 0.00 : ffff800010169988: ret 0.00 : ffff80001016998c: stp x24, x25, [x29, #56] : detach_timer(): : entry->next = LIST_POISON2; 0.00 : ffff800010169990: mov x24, #0x122 // #290 : __run_timers(): : raw_spin_lock_irq(&base->lock); 0.00 : ffff800010169994: mov x0, x20 0.00 : ffff800010169998: str x19, [x29, #16] 0.00 : ffff80001016999c: str x21, [x29, #32] : detach_timer(): : entry->next = LIST_POISON2; 0.00 : ffff8000101699a0: movk x24, #0xdead, lsl #48 0.00 : ffff8000101699a4: str x26, [x29, #72] : __run_timers(): : raw_spin_lock_irq(&base->lock); 0.00 : ffff8000101699a8: bl ffff800010cd8460 <_raw_spin_lock_irq> : base->must_forward_clk = false; 0.00 : ffff8000101699ac: strb wzr, [x20, #37] : while (time_after_eq(jiffies, base->clk)) { 0.00 : ffff8000101699b0: ldr x1, [x22, #2432] 6.64 : ffff8000101699b4: ldr x0, [x20, #16] 0.00 : ffff8000101699b8: sub x1, x1, x0 0.00 : ffff8000101699bc: tbnz x1, #63, ffff800010169b08 : __read_once_size(): 13.24 : ffff8000101699c0: ldr x19, [x22, #2432] : collect_expired_timers(): : if ((long)(now - base->clk) > 2) { 0.00 : ffff8000101699c4: sub x1, x19, x0 0.00 : ffff8000101699c8: cmp x1, #0x2 0.00 : ffff8000101699cc: b.le ffff8000101699e8 : unsigned long next = __next_timer_interrupt(base); 0.00 : ffff8000101699d0: mov x0, x20 : return 0; 0.00 : ffff8000101699d4: mov w25, #0x0 // #0 : unsigned long next = __next_timer_interrupt(base); 0.00 : ffff8000101699d8: bl ffff8000101691d8 <__next_timer_interrupt> : if (time_after(next, now)) { 0.00 : ffff8000101699dc: sub x1, x19, x0 0.00 : ffff8000101699e0: tbnz x1, #63, ffff800010169a64 : base->clk = next; 0.00 : ffff8000101699e4: str x0, [x20, #16] 0.00 : ffff8000101699e8: add x6, x20, #0x28 : __collect_expired_timers(): : int i, levels = 0; 0.00 : ffff8000101699ec: add x7, x29, #0x50 : vec = base->vectors + idx; 0.00 : ffff8000101699f0: add x10, x20, #0x70 : run_timer_softirq(): : { 0.00 : ffff8000101699f4: mov w3, #0x0 // #0 : __collect_expired_timers(): : int i, levels = 0; 0.00 : ffff8000101699f8: mov w25, #0x0 // #0 : __test_and_clear_bit(): : unsigned long mask = BIT_MASK(nr); 0.00 : ffff8000101699fc: mov x9, #0x1 // #1 : __collect_expired_timers(): : idx = (clk & LVL_MASK) + i * LVL_SIZE; 0.00 : ffff800010169a00: and w1, w0, #0x3f 0.00 : ffff800010169a04: add w1, w1, w3 : __test_and_clear_bit(): : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff800010169a08: asr w2, w1, #6 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010169a0c: lsl x5, x9, x1 : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff800010169a10: sxtw x2, w2 : unsigned long old = *p; 0.00 : ffff800010169a14: ldr x4, [x6, x2, lsl #3] : *p = old & ~mask; 0.00 : ffff800010169a18: bic x8, x4, x5 5.43 : ffff800010169a1c: str x8, [x6, x2, lsl #3] : __collect_expired_timers(): : if (__test_and_clear_bit(idx, base->pending_map)) { 0.00 : ffff800010169a20: tst x5, x4 0.00 : ffff800010169a24: b.eq ffff800010169a48 // b.none : hlist_move_list(): : new->first = old->first; 0.00 : ffff800010169a28: ldr x2, [x10, x1, lsl #3] : __collect_expired_timers(): : hlist_move_list(vec, heads++); 0.00 : ffff800010169a2c: add x4, x7, #0x8 : hlist_move_list(): 0.00 : ffff800010169a30: str x2, [x7] : if (new->first) 0.00 : ffff800010169a34: cbz x2, ffff800010169a3c : new->first->pprev = &new->first; 0.00 : ffff800010169a38: str x7, [x2, #8] : __collect_expired_timers(): : levels++; 0.00 : ffff800010169a3c: add w25, w25, #0x1 : hlist_move_list(vec, heads++); 0.00 : ffff800010169a40: mov x7, x4 : hlist_move_list(): : old->first = NULL; 0.00 : ffff800010169a44: str xzr, [x10, x1, lsl #3] : __collect_expired_timers(): : if (clk & LVL_CLK_MASK) 0.00 : ffff800010169a48: tst x0, #0x7 0.00 : ffff800010169a4c: b.ne ffff800010169a60 // b.any 0.00 : ffff800010169a50: add w3, w3, #0x40 : clk >>= LVL_CLK_SHIFT; 0.00 : ffff800010169a54: lsr x0, x0, #3 : for (i = 0; i < LVL_DEPTH; i++) { 0.00 : ffff800010169a58: cmp w3, #0x240 0.00 : ffff800010169a5c: b.ne ffff800010169a00 // b.any 1.13 : ffff800010169a60: ldr x19, [x20, #16] 0.00 : ffff800010169a64: sub w25, w25, #0x1 0.00 : ffff800010169a68: add x0, x29, #0x50 : __run_timers(): : base->clk++; 0.00 : ffff800010169a6c: add x19, x19, #0x1 1.14 : ffff800010169a70: str x19, [x20, #16] 0.00 : ffff800010169a74: add x21, x0, w25, sxtw #3 : while (levels--) 0.00 : ffff800010169a78: cmn w25, #0x1 0.00 : ffff800010169a7c: b.eq ffff8000101699b0 // b.none : __read_once_size(): 0.00 : ffff800010169a80: ldr x0, [x21] : expire_timers(): : while (!hlist_empty(head)) { 0.00 : ffff800010169a84: cbz x0, ffff800010169ad8 : timer = hlist_entry(head->first, struct timer_list, entry); 0.00 : ffff800010169a88: ldr x19, [x21] : base->running_timer = timer; 0.00 : ffff800010169a8c: str x19, [x20, #8] : __hlist_del(): : struct hlist_node **pprev = n->pprev; 0.00 : ffff800010169a90: ldp x0, x1, [x19] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010169a94: str x0, [x1] : __hlist_del(): : if (next) 0.00 : ffff800010169a98: cbz x0, ffff800010169aa0 : __write_once_size(): 0.00 : ffff800010169a9c: str x1, [x0, #8] : expire_timers(): : if (timer->flags & TIMER_IRQSAFE) { 0.00 : ffff800010169aa0: ldr w0, [x19, #32] : detach_timer(): : entry->pprev = NULL; 0.00 : ffff800010169aa4: stp x24, xzr, [x19] : expire_timers(): : fn = timer->function; 0.00 : ffff800010169aa8: ldr x26, [x19, #24] : if (timer->flags & TIMER_IRQSAFE) { 0.00 : ffff800010169aac: tbz w0, #21, ffff800010169ae4 : raw_spin_unlock(&base->lock); 0.00 : ffff800010169ab0: mov x0, x20 0.00 : ffff800010169ab4: bl ffff800010cd7c38 <_raw_spin_unlock> : call_timer_fn(timer, fn, baseclk); 0.00 : ffff800010169ab8: mov x1, x26 0.00 : ffff800010169abc: mov x0, x19 0.00 : ffff800010169ac0: bl ffff8000101693a8 : base->running_timer = NULL; 0.00 : ffff800010169ac4: str xzr, [x20, #8] : raw_spin_lock(&base->lock); 0.00 : ffff800010169ac8: mov x0, x20 0.00 : ffff800010169acc: bl ffff800010cd80f0 <_raw_spin_lock> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010169ad0: ldr x0, [x21] : expire_timers(): : while (!hlist_empty(head)) { 0.00 : ffff800010169ad4: cbnz x0, ffff800010169a88 0.00 : ffff800010169ad8: sub x21, x21, #0x8 0.00 : ffff800010169adc: sub w25, w25, #0x1 0.00 : ffff800010169ae0: b ffff800010169a78 : raw_spin_unlock_irq(&base->lock); 0.00 : ffff800010169ae4: mov x0, x20 0.00 : ffff800010169ae8: bl ffff800010cd7c70 <_raw_spin_unlock_irq> : call_timer_fn(timer, fn, baseclk); 0.00 : ffff800010169aec: mov x1, x26 0.00 : ffff800010169af0: mov x0, x19 0.00 : ffff800010169af4: bl ffff8000101693a8 : base->running_timer = NULL; 0.00 : ffff800010169af8: str xzr, [x20, #8] : raw_spin_lock_irq(&base->lock); 0.00 : ffff800010169afc: mov x0, x20 0.00 : ffff800010169b00: bl ffff800010cd8460 <_raw_spin_lock_irq> 0.00 : ffff800010169b04: b ffff800010169a80 : __run_timers(): : raw_spin_unlock_irq(&base->lock); 1.13 : ffff800010169b08: mov x0, x20 0.00 : ffff800010169b0c: bl ffff800010cd7c70 <_raw_spin_unlock_irq> 0.00 : ffff800010169b10: ldr x19, [x29, #16] 0.00 : ffff800010169b14: mov x1, sp 0.00 : ffff800010169b18: ldr x21, [x29, #32] 0.00 : ffff800010169b1c: ldp x24, x25, [x29, #56] 1.03 : ffff800010169b20: ldr x26, [x29, #72] 0.00 : ffff800010169b24: b ffff8000101697b4 0.00 : ffff800010169b28: str x19, [x29, #16] 0.00 : ffff800010169b2c: str x21, [x29, #32] 0.00 : ffff800010169b30: stp x24, x25, [x29, #56] 0.00 : ffff800010169b34: str x26, [x29, #72] : run_timer_softirq(): : } 0.00 : ffff800010169b38: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (92 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd7bf8 : : default_idle_call(): : * default_idle_call - Default CPU idle routine. : * : * To use when the cpuidle framework cannot be used. : */ : void __cpuidle default_idle_call(void) : { 5.25 : ffff800010cd7bf8: stp x29, x30, [sp, #-16]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010cd7bfc: mrs x0, sp_el0 : default_idle_call(): 0.00 : ffff800010cd7c00: mov x29, sp : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010cd7c04: ldr x0, [x0] : default_idle_call(): : if (current_clr_polling_and_test()) { 0.00 : ffff800010cd7c08: tst w0, #0x2 0.00 : ffff800010cd7c0c: b.ne ffff800010cd7c1c // b.any : local_irq_enable(); : } else { : stop_critical_timings(); : arch_cpu_idle(); 1.12 : ffff800010cd7c10: bl ffff800010087550 : start_critical_timings(); : } : } 93.63 : ffff800010cd7c14: ldp x29, x30, [sp], #16 0.00 : ffff800010cd7c18: ret : arch_local_irq_enable(): : u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1); : : WARN_ON_ONCE(pmr != GIC_PRIO_IRQON && pmr != GIC_PRIO_IRQOFF); : } : : asm volatile(ALTERNATIVE( 0.00 : ffff800010cd7c1c: mov x0, #0xe0 // #224 0.00 : ffff800010cd7c20: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010cd7c24: nop 0.00 : ffff800010cd7c28: b ffff800010cd7c14 : arch_local_irq_enable(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" ((unsigned long) GIC_PRIO_IRQON) : : "memory"); : : pmr_sync(); 0.00 : ffff800010cd7c2c: dsb sy 0.00 : ffff800010cd7c30: b ffff800010cd7c14 Percent | Source code & Disassembly of vmlinux for cycles (128 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101292d0 : : enqueue_top_rt_rq(): : { : struct rq *rq = rq_of_rt_rq(rt_rq); : : BUG_ON(&rq->rt != rt_rq); : : if (rt_rq->rt_queued) 6.99 : ffff8000101292d0: ldr w1, [x0, #1672] 0.00 : ffff8000101292d4: cbnz w1, ffff800010129330 : return; : : if (rt_rq_throttled(rt_rq)) 0.00 : ffff8000101292d8: ldr w1, [x0, #1676] 0.00 : ffff8000101292dc: cbnz w1, ffff800010129330 : { 1.59 : ffff8000101292e0: stp x29, x30, [sp, #-16]! 0.00 : ffff8000101292e4: mov x29, sp : return; : : if (rt_rq->rt_nr_running) { 0.00 : ffff8000101292e8: ldr w1, [x0, #1616] 0.00 : ffff8000101292ec: cbnz w1, ffff800010129334 : cpufreq_update_util(): : */ : static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) : { : struct update_util_data *data; : : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 10.87 : ffff8000101292f0: ldrsw x3, [x0, #2056] 0.00 : ffff8000101292f4: adrp x2, ffff800011909000 0.00 : ffff8000101292f8: add x2, x2, #0x928 0.00 : ffff8000101292fc: adrp x1, ffff80001151f000 0.76 : ffff800010129300: add x1, x1, #0x158 0.00 : ffff800010129304: ldr x2, [x2, x3, lsl #3] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 15.34 : ffff800010129308: ldr x3, [x1, x2] : cpufreq_update_util(): : cpu_of(rq))); : if (data) 0.00 : ffff80001012930c: cbz x3, ffff800010129328 0.00 : ffff800010129310: mov x1, x0 : data->func(data, rq_clock(rq), flags); 0.00 : ffff800010129314: ldr x4, [x3] 0.00 : ffff800010129318: mov w2, #0x0 // #0 0.00 : ffff80001012931c: mov x0, x3 0.00 : ffff800010129320: ldr x1, [x1, #1888] 0.00 : ffff800010129324: blr x4 : enqueue_top_rt_rq(): : rt_rq->rt_queued = 1; : } : : /* Kick cpufreq (see the comment in kernel/sched/sched.h). */ : cpufreq_update_util(rq, 0); : } 47.04 : ffff800010129328: ldp x29, x30, [sp], #16 0.00 : ffff80001012932c: ret 0.00 : ffff800010129330: ret : add_nr_running(): : unsigned prev_nr = rq->nr_running; 0.00 : ffff800010129334: sub x3, x0, #0x4, lsl #12 3.18 : ffff800010129338: ldr w2, [x3, #15876] : rq->nr_running = prev_nr + count; 0.00 : ffff80001012933c: add w1, w1, w2 6.28 : ffff800010129340: str w1, [x3, #15876] : if (prev_nr < 2 && rq->nr_running >= 2) { 0.00 : ffff800010129344: cmp w1, #0x1 0.00 : ffff800010129348: ccmp w2, #0x1, #0x2, hi // hi = pmore 0.00 : ffff80001012934c: b.ls ffff80001012935c // b.plast : enqueue_top_rt_rq(): : rt_rq->rt_queued = 1; 0.00 : ffff800010129350: mov w1, #0x1 // #1 7.11 : ffff800010129354: str w1, [x0, #1672] 0.00 : ffff800010129358: b ffff8000101292f0 : add_nr_running(): : if (!READ_ONCE(rq->rd->overload)) 0.84 : ffff80001012935c: ldr x1, [x0, #1952] : __read_once_size(): 0.00 : ffff800010129360: ldr w2, [x1, #88] : add_nr_running(): 0.00 : ffff800010129364: cbnz w2, ffff800010129350 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010129368: mov w2, #0x1 // #1 0.00 : ffff80001012936c: str w2, [x1, #88] 0.00 : ffff800010129370: b ffff800010129350 Percent | Source code & Disassembly of vmlinux for cycles (241 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107d30a0 : : nvme_submit_cmd(): : * @cmd: The command to send : * @write_sq: whether to write to the SQ doorbell : */ : static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd, : bool write_sq) : { 0.42 : ffff8000107d30a0: stp x29, x30, [sp, #-48]! 0.00 : ffff8000107d30a4: mov x29, sp 0.00 : ffff8000107d30a8: stp x19, x20, [sp, #16] 0.00 : ffff8000107d30ac: mov x19, x0 0.00 : ffff8000107d30b0: stp x21, x22, [sp, #32] : spin_lock(): : raw_spin_lock_init(&(_lock)->rlock); \ : } while (0) : : static __always_inline void spin_lock(spinlock_t *lock) : { : raw_spin_lock(&lock->rlock); 0.00 : ffff8000107d30b4: add x21, x0, #0x8 : nvme_submit_cmd(): 0.00 : ffff8000107d30b8: mov x20, x1 0.00 : ffff8000107d30bc: and w22, w2, #0xff : spin_lock(): 1.26 : ffff8000107d30c0: mov x0, x21 0.00 : ffff8000107d30c4: bl ffff800010cd80f0 <_raw_spin_lock> : nvme_submit_cmd(): : spin_lock(&nvmeq->sq_lock); : memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes), 1.26 : ffff8000107d30c8: ldrh w0, [x19, #108] 0.00 : ffff8000107d30cc: ldrb w2, [x19, #117] 9.96 : ffff8000107d30d0: ldr x1, [x19, #16] 18.78 : ffff8000107d30d4: ldp x4, x5, [x20] 0.00 : ffff8000107d30d8: lsl w0, w0, w2 0.00 : ffff8000107d30dc: add x0, x1, w0, sxtw 0.82 : ffff8000107d30e0: stp x4, x5, [x0] 4.18 : ffff8000107d30e4: ldp x2, x3, [x20, #16] 0.42 : ffff8000107d30e8: stp x2, x3, [x0, #16] 0.42 : ffff8000107d30ec: ldp x2, x3, [x20, #32] 0.42 : ffff8000107d30f0: stp x2, x3, [x0, #32] 1.25 : ffff8000107d30f4: ldp x2, x3, [x20, #48] 1.68 : ffff8000107d30f8: stp x2, x3, [x0, #48] : cmd, sizeof(*cmd)); : if (++nvmeq->sq_tail == nvmeq->q_depth) 0.00 : ffff8000107d30fc: ldrh w0, [x19, #108] 0.00 : ffff8000107d3100: ldrh w3, [x19, #104] 0.00 : ffff8000107d3104: add w0, w0, #0x1 0.00 : ffff8000107d3108: and w0, w0, #0xffff 0.00 : ffff8000107d310c: strh w0, [x19, #108] 0.00 : ffff8000107d3110: cmp w0, w3 0.00 : ffff8000107d3114: b.eq ffff8000107d31a8 // b.none : nvme_write_sq_db(): : if (!write_sq) { 0.00 : ffff8000107d3118: cbnz w22, ffff8000107d3150 : u16 next_tail = nvmeq->sq_tail + 1; 0.00 : ffff8000107d311c: add w1, w0, #0x1 : if (next_tail != nvmeq->last_sq_tail) 0.00 : ffff8000107d3120: ldrh w2, [x19, #110] : u16 next_tail = nvmeq->sq_tail + 1; 0.00 : ffff8000107d3124: and w1, w1, #0xffff : next_tail = 0; 0.00 : ffff8000107d3128: cmp w3, w1 0.00 : ffff8000107d312c: csel w1, w1, wzr, ne // ne = any : if (next_tail != nvmeq->last_sq_tail) 0.00 : ffff8000107d3130: cmp w2, w1 0.00 : ffff8000107d3134: b.eq ffff8000107d3150 // b.none : spin_unlock(): : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ : } while (0) : : static __always_inline void spin_unlock(spinlock_t *lock) : { : raw_spin_unlock(&lock->rlock); 0.00 : ffff8000107d3138: mov x0, x21 0.00 : ffff8000107d313c: bl ffff800010cd7c38 <_raw_spin_unlock> : nvme_submit_cmd(): : nvmeq->sq_tail = 0; : nvme_write_sq_db(nvmeq, write_sq); : spin_unlock(&nvmeq->sq_lock); : } 0.00 : ffff8000107d3140: ldp x19, x20, [sp, #16] 0.00 : ffff8000107d3144: ldp x21, x22, [sp, #32] 0.00 : ffff8000107d3148: ldp x29, x30, [sp], #48 0.00 : ffff8000107d314c: ret : nvme_write_sq_db(): : if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail, 0.42 : ffff8000107d3150: ldr x2, [x19, #128] : nvme_dbbuf_update_and_check_event(): : if (dbbuf_db) { 0.00 : ffff8000107d3154: cbz x2, ffff8000107d31b4 : nvme_write_sq_db(): : nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei)) 0.00 : ffff8000107d3158: ldr x3, [x19, #144] : nvme_dbbuf_update_and_check_event(): : wmb(); 0.00 : ffff8000107d315c: dsb st : old_value = *dbbuf_db; 0.00 : ffff8000107d3160: ldrh w1, [x2] : *dbbuf_db = value; 0.00 : ffff8000107d3164: str w0, [x2] : mb(); 0.00 : ffff8000107d3168: dsb sy : if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value)) 0.00 : ffff8000107d316c: ldr w2, [x3] : nvme_dbbuf_need_event(): : return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old); 0.00 : ffff8000107d3170: sub w1, w0, w1 : nvme_dbbuf_update_and_check_event(): : if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value)) 0.00 : ffff8000107d3174: and w1, w1, #0xffff : nvme_dbbuf_need_event(): : return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old); 0.00 : ffff8000107d3178: mvn w2, w2 0.00 : ffff8000107d317c: add w0, w0, w2 : nvme_dbbuf_update_and_check_event(): : if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value)) 0.00 : ffff8000107d3180: cmp w1, w0, uxth 0.00 : ffff8000107d3184: b.hi ffff8000107d31b4 // b.pmore 0.00 : ffff8000107d3188: ldrh w0, [x19, #108] : nvme_write_sq_db(): : nvmeq->last_sq_tail = nvmeq->sq_tail; 0.00 : ffff8000107d318c: strh w0, [x19, #110] : spin_unlock(): 0.84 : ffff8000107d3190: mov x0, x21 0.00 : ffff8000107d3194: bl ffff800010cd7c38 <_raw_spin_unlock> : nvme_submit_cmd(): : } 0.00 : ffff8000107d3198: ldp x19, x20, [sp, #16] 0.42 : ffff8000107d319c: ldp x21, x22, [sp, #32] 0.00 : ffff8000107d31a0: ldp x29, x30, [sp], #48 0.00 : ffff8000107d31a4: ret : nvmeq->sq_tail = 0; 0.00 : ffff8000107d31a8: strh wzr, [x19, #108] 0.00 : ffff8000107d31ac: mov w0, #0x0 // #0 0.00 : ffff8000107d31b0: b ffff8000107d3118 : nvme_write_sq_db(): : writel(nvmeq->sq_tail, nvmeq->q_db); 11.19 : ffff8000107d31b4: dmb oshst 46.28 : ffff8000107d31b8: ldrh w0, [x19, #108] : __raw_writel(): : } : : #define __raw_writel __raw_writel : static __always_inline void __raw_writel(u32 val, volatile void __iomem *addr) : { : asm volatile("str %w0, [%1]" : : "rZ" (val), "r" (addr)); 0.00 : ffff8000107d31bc: ldr x1, [x19, #96] 0.00 : ffff8000107d31c0: str w0, [x1] : nvme_write_sq_db(): : nvmeq->last_sq_tail = nvmeq->sq_tail; 0.00 : ffff8000107d31c4: strh w0, [x19, #110] 0.00 : ffff8000107d31c8: b ffff8000107d3190 Percent | Source code & Disassembly of vmlinux for cycles (230 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104662c8 : : blk_mq_get_request(): : } : : static struct request *blk_mq_get_request(struct request_queue *q, : struct bio *bio, : struct blk_mq_alloc_data *data) : { 0.85 : ffff8000104662c8: stp x29, x30, [sp, #-64]! 0.00 : ffff8000104662cc: mov x29, sp 0.44 : ffff8000104662d0: stp x19, x20, [sp, #16] 0.00 : ffff8000104662d4: mov x19, x0 0.88 : ffff8000104662d8: stp x21, x22, [sp, #32] 0.00 : ffff8000104662dc: mov x20, x2 0.00 : ffff8000104662e0: str x23, [sp, #48] 0.00 : ffff8000104662e4: mov x22, x1 : struct elevator_queue *e = q->elevator; 1.29 : ffff8000104662e8: ldr x21, [x0, #8] : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000104662ec: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.44 : ffff8000104662f0: ldr x0, [x19, #1480] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000104662f4: tst x0, #0x3 0.00 : ffff8000104662f8: b.ne ffff800010466630 // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000104662fc: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff800010466300: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010466304: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.45 : ffff800010466308: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff80001046630c: mov x3, #0x1 // #1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010466310: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff800010466314: add x0, x0, x2 0.00 : ffff800010466318: ldxr x5, [x0] 1.32 : ffff80001046631c: add x5, x5, x3 0.00 : ffff800010466320: stxr w4, x5, [x0] 0.00 : ffff800010466324: cbnz w4, ffff800010466318 : __read_once_size(): : __READ_ONCE_SIZE; 3.43 : ffff800010466328: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001046632c: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.34 : ffff800010466330: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010466334: cbnz x0, ffff8000104665a4 : percpu_ref_get_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_add(*percpu_count, nr); 0.00 : ffff800010466338: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff80001046633c: bl ffff800010160d20 <__rcu_read_unlock> : blk_mq_get_request(): : : /* alloc_time includes depth and tag waits */ : if (blk_queue_rq_alloc_time(q)) : alloc_time_ns = ktime_get_ns(); : : data->q = q; 0.00 : ffff800010466340: str x19, [x20] : if (likely(!data->ctx)) { 0.00 : ffff800010466344: ldr x0, [x20, #24] 0.00 : ffff800010466348: cbnz x0, ffff8000104665bc : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001046634c: mrs x2, tpidr_el1 : blk_mq_get_ctx(): : * care about preemption, since we know the ctx's are persistent. This does : * mean that we can't rely on ctx always matching the currently running CPU. : */ : static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q) : { : return __blk_mq_get_ctx(q, raw_smp_processor_id()); 0.00 : ffff800010466350: adrp x0, ffff80001151d000 0.00 : ffff800010466354: add x0, x0, #0x18 : __blk_mq_get_ctx(): : return per_cpu_ptr(q->queue_ctx, cpu); 0.00 : ffff800010466358: ldr w2, [x0, x2] 0.00 : ffff80001046635c: adrp x1, ffff800011909000 0.00 : ffff800010466360: add x1, x1, #0x928 0.00 : ffff800010466364: ldr x0, [x19, #56] : blk_mq_get_request(): : data->ctx = blk_mq_get_ctx(q); : clear_ctx_on_error = true; 0.00 : ffff800010466368: mov w23, #0x1 // #1 : __blk_mq_get_ctx(): 0.00 : ffff80001046636c: ldr x1, [x1, x2, lsl #3] 0.00 : ffff800010466370: add x0, x1, x0 : blk_mq_get_request(): : data->ctx = blk_mq_get_ctx(q); 0.87 : ffff800010466374: str x0, [x20, #24] : } : if (likely(!data->hctx)) 0.00 : ffff800010466378: ldr x1, [x20, #32] 0.88 : ffff80001046637c: ldr w0, [x20, #16] 0.00 : ffff800010466380: cbnz x1, ffff8000104663a0 : blk_mq_map_queue(): : type = HCTX_TYPE_POLL; 0.00 : ffff800010466384: mov w1, #0x2 // #2 : blk_mq_get_request(): : data->hctx = blk_mq_map_queue(q, data->cmd_flags, 3.49 : ffff800010466388: ldr x2, [x20, #24] : blk_mq_map_queue(): : if (flags & REQ_HIPRI) 0.00 : ffff80001046638c: tbz w0, #25, ffff800010466584 : return ctx->hctxs[type]; 0.00 : ffff800010466390: ubfiz x1, x1, #3, #2 0.00 : ffff800010466394: add x1, x2, x1 0.00 : ffff800010466398: ldr x1, [x1, #80] : blk_mq_get_request(): 0.00 : ffff80001046639c: str x1, [x20, #32] : data->ctx); : if (data->cmd_flags & REQ_NOWAIT) 0.00 : ffff8000104663a0: tbz w0, #21, ffff8000104663b0 : data->flags |= BLK_MQ_REQ_NOWAIT; 0.00 : ffff8000104663a4: ldr w1, [x20, #8] 0.00 : ffff8000104663a8: orr w1, w1, #0x1 0.00 : ffff8000104663ac: str w1, [x20, #8] : : if (e) { 0.00 : ffff8000104663b0: cbz x21, ffff8000104665f0 : data->flags |= BLK_MQ_REQ_INTERNAL; 0.00 : ffff8000104663b4: ldr w1, [x20, #8] : /* : * Flush requests are special and go directly to the : * dispatch list. Don't include reserved tags in the : * limiting, as it isn't useful. : */ : if (!op_is_flush(data->cmd_flags) && 0.00 : ffff8000104663b8: tst w0, #0x60000 : data->flags |= BLK_MQ_REQ_INTERNAL; 0.00 : ffff8000104663bc: orr w2, w1, #0x4 0.00 : ffff8000104663c0: str w2, [x20, #8] : if (!op_is_flush(data->cmd_flags) && 0.00 : ffff8000104663c4: b.ne ffff8000104663e0 // b.any : e->type->ops.limit_depth && 0.00 : ffff8000104663c8: ldr x2, [x21] 0.00 : ffff8000104663cc: ldr x2, [x2, #88] : if (!op_is_flush(data->cmd_flags) && 0.00 : ffff8000104663d0: cbz x2, ffff8000104663e0 : e->type->ops.limit_depth && 0.00 : ffff8000104663d4: tbnz w1, #1, ffff8000104663e0 : !(data->flags & BLK_MQ_REQ_RESERVED)) : e->type->ops.limit_depth(data->cmd_flags, data); 0.00 : ffff8000104663d8: mov x1, x20 0.00 : ffff8000104663dc: blr x2 : } else { : blk_mq_tag_busy(data->hctx); : } : : tag = blk_mq_get_tag(data); 7.04 : ffff8000104663e0: mov x0, x20 0.00 : ffff8000104663e4: bl ffff80001046a5d0 : if (tag == BLK_MQ_TAG_FAIL) { 1.34 : ffff8000104663e8: cmn w0, #0x1 0.00 : ffff8000104663ec: b.eq ffff800010466654 // b.none : blk_mq_tags_from_data(): : struct blk_mq_hw_ctx *hctx; : }; : : static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data) : { : if (data->flags & BLK_MQ_REQ_INTERNAL) 0.44 : ffff8000104663f0: ldr w1, [x20, #8] 0.00 : ffff8000104663f4: mov w3, w0 : blk_mq_get_request(): : data->ctx = NULL; : blk_queue_exit(q); : return NULL; : } : : rq = blk_mq_rq_ctx_init(data, tag, data->cmd_flags, alloc_time_ns); 0.00 : ffff8000104663f8: ldr w23, [x20, #16] : blk_mq_rq_ctx_init(): : struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 0.00 : ffff8000104663fc: ldr x2, [x20, #32] : blk_mq_tags_from_data(): 0.00 : ffff800010466400: tbnz w1, #2, ffff800010466568 : blk_mq_rq_ctx_init(): : struct request *rq = tags->static_rqs[tag]; 0.88 : ffff800010466404: ldr x5, [x2, #336] : req_flags_t rq_flags = 0; 0.00 : ffff800010466408: mov w1, #0x0 // #0 : if (data->hctx->flags & BLK_MQ_F_TAG_SHARED) { 1.32 : ffff80001046640c: ldr x4, [x2, #192] : struct request *rq = tags->static_rqs[tag]; 0.44 : ffff800010466410: ldr x5, [x5, #152] 7.39 : ffff800010466414: ldr x19, [x5, x3, lsl #3] : if (data->hctx->flags & BLK_MQ_F_TAG_SHARED) { 0.00 : ffff800010466418: tbnz w4, #1, ffff800010466604 : rq->internal_tag = -1; 14.79 : ffff80001046641c: mov w2, #0xffffffff // #-1 0.00 : ffff800010466420: stp w0, w2, [x19, #32] : data->hctx->tags->rqs[rq->tag] = rq; 12.61 : ffff800010466424: ldr x2, [x20, #32] 0.00 : ffff800010466428: ldr x2, [x2, #336] 0.00 : ffff80001046642c: ldr x2, [x2, #144] 0.44 : ffff800010466430: str x19, [x2, w0, sxtw #3] : rq->q = data->q; 1.31 : ffff800010466434: ldr x0, [x20] 0.00 : ffff800010466438: str x0, [x19] : rq->mq_ctx = data->ctx; 1.32 : ffff80001046643c: ldr x2, [x20, #24] 0.00 : ffff800010466440: str x2, [x19, #8] : rq->mq_hctx = data->hctx; 0.88 : ffff800010466444: ldr x2, [x20, #32] 0.00 : ffff800010466448: str x2, [x19, #16] : rq->rq_flags = rq_flags; 0.44 : ffff80001046644c: stp w23, w1, [x19, #24] : if (data->flags & BLK_MQ_REQ_PREEMPT) 0.00 : ffff800010466450: ldr w2, [x20, #8] 0.00 : ffff800010466454: tbz w2, #3, ffff800010466460 : rq->rq_flags |= RQF_PREEMPT; 0.00 : ffff800010466458: orr w1, w1, #0x100 0.00 : ffff80001046645c: str w1, [x19, #28] : if (blk_queue_io_stat(data->q)) 0.00 : ffff800010466460: ldr x2, [x20] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010466464: ldr x2, [x2, #104] : blk_mq_rq_ctx_init(): 0.00 : ffff800010466468: tst w2, #0x80 0.00 : ffff80001046646c: b.eq ffff8000104665c4 // b.none : INIT_LIST_HEAD(&rq->queuelist); 0.00 : ffff800010466470: add x0, x19, #0x48 : rq->rq_flags |= RQF_IO_STAT; 0.00 : ffff800010466474: orr w1, w1, #0x2000 : RB_CLEAR_NODE(&rq->rb_node); 0.00 : ffff800010466478: add x2, x19, #0x68 : rq->rq_flags |= RQF_IO_STAT; 0.44 : ffff80001046647c: str w1, [x19, #28] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.40 : ffff800010466480: str x0, [x19, #72] : INIT_HLIST_NODE(): : #define HLIST_HEAD_INIT { .first = NULL } : #define HLIST_HEAD(name) struct hlist_head name = { .first = NULL } : #define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL) : static inline void INIT_HLIST_NODE(struct hlist_node *h) : { : h->next = NULL; 0.00 : ffff800010466484: stp x0, xzr, [x19, #80] : blk_mq_rq_ctx_init(): : RB_CLEAR_NODE(&rq->rb_node); 0.00 : ffff800010466488: stp xzr, x2, [x19, #96] : rq->part = NULL; 0.00 : ffff80001046648c: stp xzr, xzr, [x19, #160] : ktime_get_ns(): : return ktime_mono_to_any(mono, TK_OFFS_REAL); : } : : static inline u64 ktime_get_ns(void) : { : return ktime_to_ns(ktime_get()); 0.00 : ffff800010466490: bl ffff80001016c938 : blk_mq_rq_ctx_init(): : rq->start_time_ns = ktime_get_ns(); 0.00 : ffff800010466494: str x0, [x19, #176] : rq->io_start_time_ns = 0; 0.00 : ffff800010466498: str xzr, [x19, #184] : op_is_sync(): : * PREFLUSH flag. Other operations may be marked as synchronous using the : * REQ_SYNC flag. : */ : static inline bool op_is_sync(unsigned int op) : { : return (op & REQ_OP_MASK) == REQ_OP_READ || 0.00 : ffff80001046649c: and w1, w23, #0xff : blk_mq_rq_ctx_init(): 0.00 : ffff8000104664a0: str wzr, [x19, #192] : op_is_sync(): 0.00 : ffff8000104664a4: mov w0, #0x1 // #1 : blk_mq_rq_ctx_init(): : rq->nr_phys_segments = 0; 0.00 : ffff8000104664a8: strh wzr, [x19, #196] : rq->extra_len = 0; 0.00 : ffff8000104664ac: str wzr, [x19, #204] : rq->timeout = 0; 0.00 : ffff8000104664b0: str wzr, [x19, #216] : __write_once_size(): 0.00 : ffff8000104664b4: str xzr, [x19, #224] : blk_mq_rq_ctx_init(): : rq->end_io_data = NULL; 1.26 : ffff8000104664b8: stp xzr, xzr, [x19, #264] : data->ctx->rq_dispatched[op_is_sync(op)]++; 0.00 : ffff8000104664bc: ldr x3, [x20, #24] : op_is_sync(): 0.00 : ffff8000104664c0: cbz w1, ffff8000104664d4 0.00 : ffff8000104664c4: and w23, w23, #0x7f800 0.00 : ffff8000104664c8: and w23, w23, #0xfffe0fff 0.00 : ffff8000104664cc: cmp w23, #0x0 0.00 : ffff8000104664d0: cset w0, ne // ne = any 0.00 : ffff8000104664d4: sxtw x0, w0 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104664d8: mov w2, #0x1 // #1 0.00 : ffff8000104664dc: add x0, x3, x0, lsl #3 : blk_mq_rq_ctx_init(): 0.41 : ffff8000104664e0: ldr x1, [x0, #104] 0.00 : ffff8000104664e4: add x1, x1, #0x1 3.06 : ffff8000104664e8: str x1, [x0, #104] : __write_once_size(): 0.00 : ffff8000104664ec: str w2, [x19, #212] : blk_mq_get_request(): : if (!op_is_flush(data->cmd_flags)) { 0.00 : ffff8000104664f0: ldr w0, [x20, #16] 0.00 : ffff8000104664f4: tst w0, #0x60000 0.00 : ffff8000104664f8: b.ne ffff800010466540 // b.any : rq->elv.icq = NULL; 0.00 : ffff8000104664fc: str xzr, [x19, #128] : if (e && e->type->ops.prepare_request) { 0.00 : ffff800010466500: cbz x21, ffff800010466540 0.00 : ffff800010466504: ldr x0, [x21] 0.00 : ffff800010466508: ldr x2, [x0, #96] 0.00 : ffff80001046650c: cbz x2, ffff800010466540 : if (e->type->icq_cache) 0.00 : ffff800010466510: ldr x0, [x0] 0.00 : ffff800010466514: cbz x0, ffff800010466528 : blk_mq_sched_assign_ioc(rq); 0.00 : ffff800010466518: mov x0, x19 0.00 : ffff80001046651c: bl ffff80001046c8e0 0.00 : ffff800010466520: ldr x0, [x21] 0.00 : ffff800010466524: ldr x2, [x0, #96] : : e->type->ops.prepare_request(rq, bio); 0.00 : ffff800010466528: mov x0, x19 0.00 : ffff80001046652c: mov x1, x22 0.00 : ffff800010466530: blr x2 : rq->rq_flags |= RQF_ELVPRIV; 0.00 : ffff800010466534: ldr w0, [x19, #28] 0.00 : ffff800010466538: orr w0, w0, #0x1000 0.00 : ffff80001046653c: str w0, [x19, #28] : } : } : data->hctx->queued++; 0.44 : ffff800010466540: ldr x1, [x20, #32] 0.00 : ffff800010466544: ldr x0, [x1, #352] 0.00 : ffff800010466548: add x0, x0, #0x1 0.00 : ffff80001046654c: str x0, [x1, #352] : return rq; : } 0.00 : ffff800010466550: mov x0, x19 0.00 : ffff800010466554: ldr x23, [sp, #48] 0.89 : ffff800010466558: ldp x19, x20, [sp, #16] 0.00 : ffff80001046655c: ldp x21, x22, [sp, #32] 0.00 : ffff800010466560: ldp x29, x30, [sp], #64 0.00 : ffff800010466564: ret : blk_mq_rq_ctx_init(): : struct request *rq = tags->static_rqs[tag]; 0.00 : ffff800010466568: ldr x4, [x2, #344] : req_flags_t rq_flags = 0; 0.00 : ffff80001046656c: mov w1, #0x0 // #0 : rq->tag = -1; 0.00 : ffff800010466570: mov w2, #0xffffffff // #-1 : struct request *rq = tags->static_rqs[tag]; 0.00 : ffff800010466574: ldr x4, [x4, #152] 0.00 : ffff800010466578: ldr x19, [x4, x3, lsl #3] : rq->internal_tag = tag; 0.00 : ffff80001046657c: stp w2, w0, [x19, #32] 0.00 : ffff800010466580: b ffff800010466434 : blk_mq_map_queue(): : else if ((flags & REQ_OP_MASK) == REQ_OP_READ) 3.50 : ffff800010466584: and w1, w0, #0xff 0.00 : ffff800010466588: cmp w1, #0x0 0.00 : ffff80001046658c: cset w1, eq // eq = none : return ctx->hctxs[type]; 0.00 : ffff800010466590: ubfiz x1, x1, #3, #2 0.00 : ffff800010466594: add x1, x2, x1 0.00 : ffff800010466598: ldr x1, [x1, #80] : blk_mq_get_request(): : data->hctx = blk_mq_map_queue(q, data->cmd_flags, 5.27 : ffff80001046659c: str x1, [x20, #32] 0.00 : ffff8000104665a0: b ffff8000104663a0 : __read_once_size(): : __READ_ONCE_SIZE; 0.88 : ffff8000104665a4: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000104665a8: cbz x0, ffff800010466338 : rcu_read_unlock(): 9.58 : ffff8000104665ac: bl ffff800010160d20 <__rcu_read_unlock> : blk_mq_get_request(): : data->q = q; 0.00 : ffff8000104665b0: str x19, [x20] : if (likely(!data->ctx)) { 0.89 : ffff8000104665b4: ldr x0, [x20, #24] 0.00 : ffff8000104665b8: cbz x0, ffff80001046634c : bool clear_ctx_on_error = false; 0.00 : ffff8000104665bc: mov w23, #0x0 // #0 0.00 : ffff8000104665c0: b ffff800010466378 : blk_mq_rq_ctx_init(): : INIT_LIST_HEAD(&rq->queuelist); 0.00 : ffff8000104665c4: add x1, x19, #0x48 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000104665c8: str x1, [x19, #72] : INIT_HLIST_NODE(): 0.00 : ffff8000104665cc: stp x1, xzr, [x19, #80] : blk_mq_rq_ctx_init(): : RB_CLEAR_NODE(&rq->rb_node); 0.00 : ffff8000104665d0: add x1, x19, #0x68 : INIT_HLIST_NODE(): : h->pprev = NULL; 0.00 : ffff8000104665d4: str xzr, [x19, #96] : blk_mq_rq_ctx_init(): 0.00 : ffff8000104665d8: str x1, [x19, #104] : rq->part = NULL; 0.00 : ffff8000104665dc: stp xzr, xzr, [x19, #160] : blk_mq_need_time_stamp(): : return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator; 0.00 : ffff8000104665e0: ldr x0, [x0, #8] 0.00 : ffff8000104665e4: cbnz x0, ffff800010466490 : blk_mq_rq_ctx_init(): : rq->start_time_ns = 0; 0.00 : ffff8000104665e8: str xzr, [x19, #176] 0.00 : ffff8000104665ec: b ffff800010466498 : blk_mq_get_request(): : blk_mq_tag_busy(data->hctx); 0.43 : ffff8000104665f0: ldr x0, [x20, #32] : blk_mq_tag_busy(): : extern bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *); : extern void __blk_mq_tag_idle(struct blk_mq_hw_ctx *); : : static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) : { : if (!(hctx->flags & BLK_MQ_F_TAG_SHARED)) 6.24 : ffff8000104665f4: ldr x1, [x0, #192] 0.00 : ffff8000104665f8: tbz w1, #1, ffff8000104663e0 : return false; : : return __blk_mq_tag_busy(hctx); 0.00 : ffff8000104665fc: bl ffff80001046a440 <__blk_mq_tag_busy> 0.00 : ffff800010466600: b ffff8000104663e0 : blk_mq_rq_ctx_init(): : atomic_inc(&data->hctx->nr_active); 0.00 : ffff800010466604: add x3, x2, #0x1b0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010466608: b ffff800010466620 0.00 : ffff80001046660c: b ffff800010466620 : __lse_atomic_add(): : } : : ATOMIC_OP(andnot, stclr) : ATOMIC_OP(or, stset) : ATOMIC_OP(xor, steor) : ATOMIC_OP(add, stadd) 0.00 : ffff800010466610: mov w1, #0x1 // #1 0.00 : ffff800010466614: stadd w1, [x3] : blk_mq_rq_ctx_init(): : rq_flags = RQF_MQ_INFLIGHT; 0.00 : ffff800010466618: mov w1, #0x40 // #64 0.00 : ffff80001046661c: b ffff80001046641c : __ll_sc_atomic_add(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff800010466620: add x2, x2, #0x1b0 0.00 : ffff800010466624: b ffff800010469eec : blk_mq_rq_ctx_init(): 0.00 : ffff800010466628: mov w1, #0x40 // #64 0.00 : ffff80001046662c: b ffff80001046641c : percpu_ref_get_many(): : else : atomic_long_add(nr, &ref->count); 0.00 : ffff800010466630: add x1, x19, #0x5c0 : arch_static_branch_jump(): 0.00 : ffff800010466634: b ffff800010466648 0.00 : ffff800010466638: b ffff800010466648 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff80001046663c: mov x0, #0x1 // #1 0.00 : ffff800010466640: stadd x0, [x1] 0.00 : ffff800010466644: b ffff80001046633c : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff800010466648: add x2, x19, #0x5c0 0.00 : ffff80001046664c: b ffff800010469f04 0.00 : ffff800010466650: b ffff80001046633c : blk_mq_get_request(): : if (clear_ctx_on_error) 0.00 : ffff800010466654: cbz w23, ffff80001046665c : data->ctx = NULL; 0.00 : ffff800010466658: str xzr, [x20, #24] : blk_queue_exit(q); 0.00 : ffff80001046665c: mov x0, x19 : return NULL; 0.00 : ffff800010466660: mov x19, #0x0 // #0 : blk_queue_exit(q); 0.00 : ffff800010466664: bl ffff80001045bb30 : return NULL; 0.00 : ffff800010466668: b ffff800010466550 Percent | Source code & Disassembly of vmlinux for cycles (83 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015f008 : : note_gp_changes(): : rcu_gpnum_ovf(rnp, rdp); : return ret; : } : : static void note_gp_changes(struct rcu_data *rdp) : { 0.00 : ffff80001015f008: stp x29, x30, [sp, #-48]! 0.00 : ffff80001015f00c: mov x29, sp 2.46 : ffff80001015f010: stp x20, x21, [sp, #24] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001015f014: mrs x20, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 1.16 : ffff80001015f018: and w1, w20, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff80001015f01c: cbnz w1, ffff80001015f028 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001015f020: mov x1, #0x60 // #96 0.00 : ffff80001015f024: msr daifset, #0x2 : note_gp_changes(): : unsigned long flags; : bool needwake; : struct rcu_node *rnp; : : local_irq_save(flags); : rnp = rdp->mynode; 7.27 : ffff80001015f028: ldr x21, [x0, #24] : if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) && 7.14 : ffff80001015f02c: ldr x2, [x0] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001015f030: ldr x1, [x21, #8] : note_gp_changes(): 0.00 : ffff80001015f034: cmp x2, x1 0.00 : ffff80001015f038: b.ne ffff80001015f05c // b.any : __read_once_size(): 20.67 : ffff80001015f03c: ldrb w1, [x0, #20] : note_gp_changes(): 0.00 : ffff80001015f040: tst w1, #0xff 0.00 : ffff80001015f044: b.ne ffff80001015f05c // b.any : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 1.28 : ffff80001015f048: msr daif, x20 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 10.73 : ffff80001015f04c: nop : note_gp_changes(): : } : needwake = __note_gp_changes(rnp, rdp); : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); : if (needwake) : rcu_gp_kthread_wake(); : } 0.00 : ffff80001015f050: ldp x20, x21, [sp, #24] 0.00 : ffff80001015f054: ldp x29, x30, [sp], #48 0.00 : ffff80001015f058: ret 8.49 : ffff80001015f05c: str x19, [x29, #16] 2.44 : ffff80001015f060: mov x19, x0 : !raw_spin_trylock_rcu_node(rnp)) { /* irqs already off, so later. */ 0.00 : ffff80001015f064: mov x0, x21 0.00 : ffff80001015f068: bl ffff800010cd7fe0 <_raw_spin_trylock> : !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */ 0.00 : ffff80001015f06c: cbnz w0, ffff80001015f088 0.00 : ffff80001015f070: ldr x19, [x29, #16] 0.00 : ffff80001015f074: b ffff80001015f048 : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff80001015f078: dsb sy : note_gp_changes(): : } 31.03 : ffff80001015f07c: ldp x20, x21, [sp, #24] 7.34 : ffff80001015f080: ldp x29, x30, [sp], #48 0.00 : ffff80001015f084: ret : needwake = __note_gp_changes(rnp, rdp); 0.00 : ffff80001015f088: mov x1, x19 0.00 : ffff80001015f08c: mov x0, x21 0.00 : ffff80001015f090: bl ffff80001015ee98 <__note_gp_changes> 0.00 : ffff80001015f094: and w19, w0, #0xff : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff80001015f098: mov x1, x20 0.00 : ffff80001015f09c: mov x0, x21 0.00 : ffff80001015f0a0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : if (needwake) 0.00 : ffff80001015f0a4: cbnz w19, ffff80001015f0b8 0.00 : ffff80001015f0a8: ldr x19, [x29, #16] : } 0.00 : ffff80001015f0ac: ldp x20, x21, [sp, #24] 0.00 : ffff80001015f0b0: ldp x29, x30, [sp], #48 0.00 : ffff80001015f0b4: ret : rcu_gp_kthread_wake(); 0.00 : ffff80001015f0b8: bl ffff80001015ccd0 0.00 : ffff80001015f0bc: ldr x19, [x29, #16] 0.00 : ffff80001015f0c0: b ffff80001015f050 Percent | Source code & Disassembly of vmlinux for cycles (80 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011b948 : : account_process_tick(): : * Account a single tick of CPU time. : * @p: the process that the CPU time gets accounted to : * @user_tick: indicates if the tick is a user or a system tick : */ : void account_process_tick(struct task_struct *p, int user_tick) : { 0.00 : ffff80001011b948: stp x29, x30, [sp, #-32]! : u64 cputime, steal; : : if (vtime_accounting_enabled_this_cpu()) : return; : : if (sched_clock_irqtime) { 0.00 : ffff80001011b94c: adrp x2, ffff800011afb000 : { 0.00 : ffff80001011b950: mov x29, sp : if (sched_clock_irqtime) { 1.30 : ffff80001011b954: ldr w2, [x2, #3992] 0.00 : ffff80001011b958: cbnz w2, ffff80001011ba68 0.00 : ffff80001011b95c: stp x19, x20, [x29, #16] 0.00 : ffff80001011b960: mov x19, x0 0.00 : ffff80001011b964: mov w20, w1 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011b968: nop 0.00 : ffff80001011b96c: mov x2, #0x900 // #2304 0.00 : ffff80001011b970: movk x2, #0x3d, lsl #16 : account_process_tick(): : if (steal >= cputime) : return; : : cputime -= steal; : : if (user_tick) 0.00 : ffff80001011b974: cbnz w20, ffff80001011ba44 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011b978: mrs x0, tpidr_el1 : account_process_tick(): : account_user_time(p, cputime); : else if ((p != this_rq()->idle) || (irq_count() != HARDIRQ_OFFSET)) 0.00 : ffff80001011b97c: adrp x3, ffff800011528000 0.00 : ffff80001011b980: add x3, x3, #0xe80 0.00 : ffff80001011b984: add x3, x3, x0 0.00 : ffff80001011b988: ldr x0, [x3, #2360] 0.00 : ffff80001011b98c: cmp x0, x19 0.00 : ffff80001011b990: b.ne ffff80001011b9a8 // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001011b994: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011b998: ldr w0, [x0, #16] : account_process_tick(): 0.00 : ffff80001011b99c: and w0, w0, #0x1fff00 0.00 : ffff80001011b9a0: cmp w0, #0x10, lsl #12 0.00 : ffff80001011b9a4: b.eq ffff80001011ba78 // b.none : account_system_time(p, HARDIRQ_OFFSET, cputime); 0.00 : ffff80001011b9a8: mov x0, x19 0.00 : ffff80001011b9ac: mov w1, #0x10000 // #65536 0.00 : ffff80001011b9b0: bl ffff80001011b588 0.00 : ffff80001011b9b4: ldp x19, x20, [x29, #16] : else : account_idle_time(cputime); : } 0.00 : ffff80001011b9b8: ldp x29, x30, [sp], #32 0.00 : ffff80001011b9bc: ret : paravirt_steal_clock(): : : extern struct paravirt_patch_template pv_ops; : : static inline u64 paravirt_steal_clock(int cpu) : { : return pv_ops.time.steal_clock(cpu); 0.00 : ffff80001011b9c0: adrp x1, ffff800011af8000 : steal_account_process_time(): : steal = paravirt_steal_clock(smp_processor_id()); 0.00 : ffff80001011b9c4: adrp x0, ffff80001151d000 : __my_cpu_offset(): 0.00 : ffff80001011b9c8: mrs x2, tpidr_el1 : paravirt_steal_clock(): 0.00 : ffff80001011b9cc: ldr x1, [x1, #248] : steal_account_process_time(): 0.00 : ffff80001011b9d0: add x0, x0, #0x18 : paravirt_steal_clock(): 0.00 : ffff80001011b9d4: ldr w0, [x0, x2] 0.00 : ffff80001011b9d8: blr x1 : __my_cpu_offset(): : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff80001011b9dc: mov x4, sp : steal_account_process_time(): : steal -= this_rq()->prev_steal_time; 0.00 : ffff80001011b9e0: adrp x3, ffff800011528000 0.00 : ffff80001011b9e4: add x3, x3, #0xe80 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011b9e8: mrs x2, tpidr_el1 : steal_account_process_time(): 0.00 : ffff80001011b9ec: add x6, x3, x2 : account_steal_time(): : u64 *cpustat = kcpustat_this_cpu->cpustat; 0.00 : ffff80001011b9f0: adrp x1, ffff80001151e000 0.00 : ffff80001011b9f4: add x1, x1, #0xfc0 : account_process_tick(): : if (steal >= cputime) 0.00 : ffff80001011b9f8: mov x5, #0x8ff // #2303 : account_steal_time(): : u64 *cpustat = kcpustat_this_cpu->cpustat; 0.00 : ffff80001011b9fc: add x1, x1, x2 : steal_account_process_time(): : steal -= this_rq()->prev_steal_time; 0.00 : ffff80001011ba00: ldr x2, [x6, #2848] : account_process_tick(): : if (steal >= cputime) 0.00 : ffff80001011ba04: movk x5, #0x3d, lsl #16 : steal_account_process_time(): : steal -= this_rq()->prev_steal_time; 0.00 : ffff80001011ba08: sub x0, x0, x2 : account_steal_time(): : cpustat[CPUTIME_STEAL] += cputime; 0.00 : ffff80001011ba0c: ldr x2, [x1, #56] : account_process_tick(): : if (steal >= cputime) 0.00 : ffff80001011ba10: cmp x0, x5 : account_steal_time(): : cpustat[CPUTIME_STEAL] += cputime; 0.00 : ffff80001011ba14: add x2, x2, x0 0.00 : ffff80001011ba18: str x2, [x1, #56] : __my_cpu_offset(): 0.00 : ffff80001011ba1c: mrs x1, tpidr_el1 : steal_account_process_time(): : this_rq()->prev_steal_time += steal; 0.00 : ffff80001011ba20: add x3, x3, x1 0.00 : ffff80001011ba24: ldr x1, [x3, #2848] 0.00 : ffff80001011ba28: add x1, x1, x0 0.00 : ffff80001011ba2c: str x1, [x3, #2848] : account_process_tick(): : if (steal >= cputime) 0.00 : ffff80001011ba30: b.hi ffff80001011ba5c // b.pmore 0.00 : ffff80001011ba34: mov x2, #0x900 // #2304 0.00 : ffff80001011ba38: movk x2, #0x3d, lsl #16 0.00 : ffff80001011ba3c: sub x2, x2, x0 : if (user_tick) 0.00 : ffff80001011ba40: cbz w20, ffff80001011b978 : account_user_time(p, cputime); 0.00 : ffff80001011ba44: mov x0, x19 0.00 : ffff80001011ba48: mov x1, x2 0.00 : ffff80001011ba4c: bl ffff80001011b308 0.00 : ffff80001011ba50: ldp x19, x20, [x29, #16] : } 0.00 : ffff80001011ba54: ldp x29, x30, [sp], #32 0.00 : ffff80001011ba58: ret 0.00 : ffff80001011ba5c: ldp x19, x20, [x29, #16] 0.00 : ffff80001011ba60: ldp x29, x30, [sp], #32 0.00 : ffff80001011ba64: ret : irqtime_account_process_tick(p, user_tick, 1); 98.70 : ffff80001011ba68: mov w2, #0x1 // #1 0.00 : ffff80001011ba6c: bl ffff80001011b668 : } 0.00 : ffff80001011ba70: ldp x29, x30, [sp], #32 0.00 : ffff80001011ba74: ret : account_idle_time(cputime); 0.00 : ffff80001011ba78: mov x0, x2 0.00 : ffff80001011ba7c: bl ffff80001011b620 0.00 : ffff80001011ba80: ldp x19, x20, [x29, #16] 0.00 : ffff80001011ba84: b ffff80001011ba60 Percent | Source code & Disassembly of vmlinux for cycles (84 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010129eb8 : : task_woken_rt(): : static void task_woken_rt(struct rq *rq, struct task_struct *p) : { : bool need_to_push = !task_running(rq, p) && : !test_tsk_need_resched(rq->curr) && : p->nr_cpus_allowed > 1 && : (dl_task(rq->curr) || rt_task(rq->curr)) && 7.37 : ffff800010129eb8: ldr w2, [x1, #64] 0.00 : ffff800010129ebc: cbnz w2, ffff800010129ee8 : !test_tsk_need_resched(rq->curr) && 32.31 : ffff800010129ec0: ldr x2, [x0, #2352] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 31.93 : ffff800010129ec4: ldr x3, [x2] : task_woken_rt(): : bool need_to_push = !task_running(rq, p) && 0.00 : ffff800010129ec8: tst w3, #0x2 0.00 : ffff800010129ecc: b.ne ffff800010129ee8 // b.any : !test_tsk_need_resched(rq->curr) && 0.00 : ffff800010129ed0: ldr w3, [x1, #732] 0.00 : ffff800010129ed4: cmp w3, #0x1 0.00 : ffff800010129ed8: b.le ffff800010129ee8 : dl_task(): : return 0; : } : : static inline int dl_task(struct task_struct *p) : { : return dl_prio(p->prio); 0.00 : ffff800010129edc: ldr w3, [x2, #108] : rt_prio(): : : struct task_struct; : : static inline int rt_prio(int prio) : { : if (unlikely(prio < MAX_RT_PRIO)) 0.00 : ffff800010129ee0: cmp w3, #0x63 0.00 : ffff800010129ee4: b.le ffff800010129eec 28.38 : ffff800010129ee8: ret : task_woken_rt(): : { 0.00 : ffff800010129eec: stp x29, x30, [sp, #-32]! 0.00 : ffff800010129ef0: mov x29, sp 0.00 : ffff800010129ef4: str x19, [sp, #16] 0.00 : ffff800010129ef8: mov x19, x0 : (dl_task(rq->curr) || rt_task(rq->curr)) && 0.00 : ffff800010129efc: ldr w0, [x2, #732] 0.00 : ffff800010129f00: cmp w0, #0x1 0.00 : ffff800010129f04: b.le ffff800010129f18 : (rq->curr->nr_cpus_allowed < 2 || 0.00 : ffff800010129f08: ldr w0, [x1, #108] 0.00 : ffff800010129f0c: cmp w0, w3 0.00 : ffff800010129f10: b.lt ffff800010129f24 // b.tstop 0.00 : ffff800010129f14: nop : push_rt_tasks(): : while (push_rt_task(rq)) 0.00 : ffff800010129f18: mov x0, x19 0.00 : ffff800010129f1c: bl ffff800010129b18 0.00 : ffff800010129f20: cbnz w0, ffff800010129f18 : task_woken_rt(): : rq->curr->prio <= p->prio); : : if (need_to_push || !rt_task_fits_capacity(p, cpu_of(rq))) : push_rt_tasks(rq); : } 0.00 : ffff800010129f24: ldr x19, [sp, #16] 0.00 : ffff800010129f28: ldp x29, x30, [sp], #32 0.00 : ffff800010129f2c: ret Percent | Source code & Disassembly of vmlinux for cycles (237 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102c6698 : : blkdev_direct_IO(): : return ret; : } : : static ssize_t : blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) : { 0.00 : ffff8000102c6698: stp x29, x30, [sp, #-176]! 0.00 : ffff8000102c669c: mov x29, sp 3.00 : ffff8000102c66a0: str x20, [sp, #24] 0.00 : ffff8000102c66a4: mov x20, x0 0.86 : ffff8000102c66a8: str x23, [sp, #48] 0.00 : ffff8000102c66ac: mov x23, x1 0.85 : ffff8000102c66b0: str x25, [sp, #64] 0.00 : ffff8000102c66b4: adrp x1, ffff800011909000 0.00 : ffff8000102c66b8: add x1, x1, #0x908 0.43 : ffff8000102c66bc: ldr x0, [x1] 0.43 : ffff8000102c66c0: str x0, [x29, #168] 0.00 : ffff8000102c66c4: mov x0, #0x0 // #0 : int nr_pages; : : nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1); 0.00 : ffff8000102c66c8: mov w1, #0x101 // #257 0.00 : ffff8000102c66cc: mov x0, x23 1.72 : ffff8000102c66d0: bl ffff800010489938 0.00 : ffff8000102c66d4: mov w2, w0 : if (!nr_pages) : return 0; 0.00 : ffff8000102c66d8: mov x0, #0x0 // #0 : if (!nr_pages) 0.00 : ffff8000102c66dc: cbz w2, ffff8000102c6980 0.00 : ffff8000102c66e0: str x24, [x29, #56] : if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES) 0.00 : ffff8000102c66e4: ldr x0, [x20, #16] 0.00 : ffff8000102c66e8: cmp x0, #0x0 0.00 : ffff8000102c66ec: cset w0, eq // eq = none 0.00 : ffff8000102c66f0: cmp w2, #0x100 4.23 : ffff8000102c66f4: cset w24, le 0.00 : ffff8000102c66f8: ands w24, w0, w24 0.00 : ffff8000102c66fc: b.ne ffff8000102c6a4c // b.any 0.00 : ffff8000102c6700: str x19, [x29, #16] : return __blkdev_direct_IO_simple(iocb, iter, nr_pages); : : return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES)); 0.00 : ffff8000102c6704: cmp w2, #0x100 1.71 : ffff8000102c6708: str x22, [x29, #40] 0.00 : ffff8000102c670c: mov w19, #0x100 // #256 0.00 : ffff8000102c6710: str x26, [x29, #72] 0.00 : ffff8000102c6714: csel w19, w2, w19, le 0.00 : ffff8000102c6718: str x28, [x29, #88] : __blkdev_direct_IO(): : loff_t pos = iocb->ki_pos; 0.00 : ffff8000102c671c: ldp x0, x22, [x20] : bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0; 0.43 : ffff8000102c6720: ldr w4, [x20, #32] : iov_iter_rw(): : return iov_iter_type(i) == ITER_DISCARD; : } : : static inline unsigned char iov_iter_rw(const struct iov_iter *i) : { : return i->type & (READ | WRITE); 0.00 : ffff8000102c6724: ldr w26, [x23] : bdev_file_inode(): : return file->f_mapping->host; 0.00 : ffff8000102c6728: ldr x3, [x0, #240] : __blkdev_direct_IO(): : bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0; 0.00 : ffff8000102c672c: and w0, w4, #0x8 0.42 : ffff8000102c6730: str w0, [x29, #120] : iov_iter_rw(): 0.00 : ffff8000102c6734: and w0, w26, #0x1 0.43 : ffff8000102c6738: str w0, [x29, #124] : __blkdev_direct_IO(): : if ((pos | iov_iter_alignment(iter)) & 0.00 : ffff8000102c673c: mov x0, x23 : bdev_file_inode(): : return file->f_mapping->host; 0.00 : ffff8000102c6740: ldr x28, [x3] : __blkdev_direct_IO(): : if ((pos | iov_iter_alignment(iter)) & 0.00 : ffff8000102c6744: bl ffff800010489108 0.00 : ffff8000102c6748: orr x0, x22, x0 : queue_logical_block_size(): : : static inline unsigned queue_logical_block_size(const struct request_queue *q) : { : int retval = 512; : : if (q && q->limits.logical_block_size) 0.00 : ffff8000102c674c: mov x1, #0x1ff // #511 : bdev_get_queue(): : return bdev->bd_disk->queue; /* this is never NULL */ 0.00 : ffff8000102c6750: ldur x2, [x28, #-88] 1.26 : ffff8000102c6754: ldr x2, [x2, #1040] : queue_logical_block_size(): : if (q && q->limits.logical_block_size) 0.00 : ffff8000102c6758: cbz x2, ffff8000102c676c 8.09 : ffff8000102c675c: ldr w2, [x2, #1088] 0.00 : ffff8000102c6760: sub w3, w2, #0x1 0.00 : ffff8000102c6764: cmp w2, #0x0 0.00 : ffff8000102c6768: csel x1, x3, x1, ne // ne = any : __blkdev_direct_IO(): 9.56 : ffff8000102c676c: tst x0, x1 0.00 : ffff8000102c6770: b.ne ffff8000102c6b34 // b.any : bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, &blkdev_dio_pool); 0.00 : ffff8000102c6774: mov w1, w19 2.42 : ffff8000102c6778: str x21, [x29, #32] 0.00 : ffff8000102c677c: adrp x2, ffff800011b32000 0.00 : ffff8000102c6780: mov w0, #0xcc0 // #3264 0.00 : ffff8000102c6784: add x2, x2, #0x828 0.00 : ffff8000102c6788: str x27, [x29, #80] 0.00 : ffff8000102c678c: bl ffff800010456f78 0.00 : ffff8000102c6790: mov x21, x0 : dio = container_of(bio, struct blkdev_dio, bio); 0.00 : ffff8000102c6794: sub x19, x0, #0x18 : dio->is_sync = is_sync = is_sync_kiocb(iocb); 0.00 : ffff8000102c6798: ldr x0, [x20, #16] 0.00 : ffff8000102c679c: mov x1, x0 0.00 : ffff8000102c67a0: ldurb w0, [x21, #-4] : is_sync_kiocb(): : randomized_struct_fields_end : }; : : static inline bool is_sync_kiocb(struct kiocb *kiocb) : { : return kiocb->ki_complete == NULL; 0.00 : ffff8000102c67a4: cmp x1, #0x0 : __blkdev_direct_IO(): 0.00 : ffff8000102c67a8: str x1, [x29, #104] : is_sync_kiocb(): 0.00 : ffff8000102c67ac: cset w1, eq // eq = none : __blkdev_direct_IO(): 0.00 : ffff8000102c67b0: bfi w0, w1, #2, #1 0.00 : ffff8000102c67b4: sturb w0, [x21, #-4] : if (dio->is_sync) { 0.00 : ffff8000102c67b8: tbnz w0, #2, ffff8000102c6aa8 : dio->iocb = iocb; 0.00 : ffff8000102c67bc: stur x20, [x21, #-24] : dio->multi_bio = false; 0.86 : ffff8000102c67c0: ldurb w0, [x21, #-4] : dio->size = 0; 3.00 : ffff8000102c67c4: stur xzr, [x21, #-16] : dio->multi_bio = false; 0.00 : ffff8000102c67c8: and w0, w0, #0xfffffffe 0.00 : ffff8000102c67cc: sturb w0, [x21, #-4] : dio->should_dirty = is_read && iter_is_iovec(iter); 3.81 : ffff8000102c67d0: ldr w0, [x29, #124] 0.00 : ffff8000102c67d4: cbnz w0, ffff8000102c67e8 : iov_iter_type(): : return i->type & ~(READ | WRITE); 0.00 : ffff8000102c67d8: ldr w0, [x23] 0.00 : ffff8000102c67dc: and w0, w0, #0xfffffffe : __blkdev_direct_IO(): 0.00 : ffff8000102c67e0: cmp w0, #0x4 0.00 : ffff8000102c67e4: cset w24, eq // eq = none 2.46 : ffff8000102c67e8: ldurb w0, [x21, #-4] 0.00 : ffff8000102c67ec: bfi w0, w24, #1, #1 8.01 : ffff8000102c67f0: sturb w0, [x21, #-4] : if (!is_poll) 0.00 : ffff8000102c67f4: ldr w0, [x29, #120] 0.00 : ffff8000102c67f8: cbz w0, ffff8000102c6a9c 0.00 : ffff8000102c67fc: adrp x27, ffff8000102c6000 : bio_alloc(): : : extern struct bio_set fs_bio_set; : : static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) : { : return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set); 0.00 : ffff8000102c6800: adrp x0, ffff800011b3a000 : queue_logical_block_size(): 0.00 : ffff8000102c6804: mov x26, x21 : bio_alloc(): 0.00 : ffff8000102c6808: add x0, x0, #0x2b8 0.00 : ffff8000102c680c: sub x24, x28, #0xd8 : __blkdev_direct_IO(): : bio->bi_end_io = blkdev_bio_end_io; 0.00 : ffff8000102c6810: add x27, x27, #0xc58 : bio_alloc(): 0.39 : ffff8000102c6814: str x0, [x29, #112] 0.00 : ffff8000102c6818: b ffff8000102c68b8 : __blkdev_direct_IO(): : bio->bi_opf = REQ_OP_READ; 1.64 : ffff8000102c681c: str wzr, [x26, #16] : if (dio->should_dirty) 3.82 : ffff8000102c6820: ldrb w0, [x19, #20] 0.00 : ffff8000102c6824: tbnz w0, #1, ffff8000102c69d8 : dio->size += bio->bi_iter.bi_size; 0.00 : ffff8000102c6828: ldr w4, [x26, #40] : nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES); 0.00 : ffff8000102c682c: mov w1, #0x100 // #256 : dio->size += bio->bi_iter.bi_size; 0.43 : ffff8000102c6830: ldr x3, [x19, #8] : nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES); 0.00 : ffff8000102c6834: mov x0, x23 : dio->size += bio->bi_iter.bi_size; 0.00 : ffff8000102c6838: add x3, x3, x4 0.00 : ffff8000102c683c: str x3, [x19, #8] : pos += bio->bi_iter.bi_size; 0.00 : ffff8000102c6840: ldr w3, [x26, #40] 0.00 : ffff8000102c6844: add x22, x22, x3 : nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES); 0.00 : ffff8000102c6848: bl ffff800010489938 0.43 : ffff8000102c684c: mov w25, w0 : if (!nr_pages) { 0.00 : ffff8000102c6850: cbz w0, ffff8000102c6a60 : if (!dio->multi_bio) { 0.00 : ffff8000102c6854: ldrb w0, [x19, #20] 0.00 : ffff8000102c6858: tbnz w0, #0, ffff8000102c69b4 : if (!is_sync) 0.00 : ffff8000102c685c: ldr x0, [x29, #104] 0.00 : ffff8000102c6860: cbz x0, ffff8000102c6888 : bio_get(): : bio->bi_flags |= (1 << BIO_REFFED); 0.00 : ffff8000102c6864: ldrh w0, [x26, #20] 0.00 : ffff8000102c6868: orr w0, w0, #0x100 0.00 : ffff8000102c686c: strh w0, [x26, #20] : smp_mb__before_atomic(); 0.00 : ffff8000102c6870: dmb ish : atomic_inc(&bio->__bi_cnt); 0.00 : ffff8000102c6874: add x0, x26, #0x64 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102c6878: b ffff8000102c69ac 0.00 : ffff8000102c687c: b ffff8000102c69ac : __lse_atomic_add(): : } : : ATOMIC_OP(andnot, stclr) : ATOMIC_OP(or, stset) : ATOMIC_OP(xor, steor) : ATOMIC_OP(add, stadd) 0.00 : ffff8000102c6880: mov w1, #0x1 // #1 0.00 : ffff8000102c6884: stadd w1, [x0] : __blkdev_direct_IO(): : dio->multi_bio = true; 0.00 : ffff8000102c6888: ldrb w0, [x19, #20] : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102c688c: mov w1, #0x2 // #2 : __blkdev_direct_IO(): 0.00 : ffff8000102c6890: orr w0, w0, #0x1 0.00 : ffff8000102c6894: strb w0, [x19, #20] : __write_once_size(): 0.00 : ffff8000102c6898: stur w1, [x21, #-8] : __blkdev_direct_IO(): : submit_bio(bio); 0.00 : ffff8000102c689c: mov x0, x26 0.00 : ffff8000102c68a0: bl ffff80001045b7e0 : bio_alloc(): : return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set); 0.00 : ffff8000102c68a4: ldr x2, [x29, #112] 0.00 : ffff8000102c68a8: mov w1, w25 0.00 : ffff8000102c68ac: mov w0, #0xcc0 // #3264 0.00 : ffff8000102c68b0: bl ffff800010456f78 0.00 : ffff8000102c68b4: mov x26, x0 : __blkdev_direct_IO(): : bio_set_dev(bio, bdev); 0.42 : ffff8000102c68b8: ldr x0, [x24, #128] 1.28 : ffff8000102c68bc: ldr x1, [x26, #8] 0.00 : ffff8000102c68c0: cmp x1, x0 0.00 : ffff8000102c68c4: b.eq ffff8000102c68d8 // b.none : bio_clear_flag(): : bio->bi_flags &= ~(1U << bit); 0.40 : ffff8000102c68c8: ldrh w0, [x26, #20] 0.00 : ffff8000102c68cc: and w0, w0, #0xfffffdff 0.82 : ffff8000102c68d0: strh w0, [x26, #20] 0.43 : ffff8000102c68d4: ldr x0, [x24, #128] : __blkdev_direct_IO(): 0.00 : ffff8000102c68d8: str x0, [x26, #8] 0.00 : ffff8000102c68dc: mov x0, x26 0.43 : ffff8000102c68e0: ldrb w1, [x24, #108] 0.43 : ffff8000102c68e4: strb w1, [x26, #27] 0.00 : ffff8000102c68e8: bl ffff800010456970 : bio->bi_iter.bi_sector = pos >> 9; 0.85 : ffff8000102c68ec: asr x0, x22, #9 1.25 : ffff8000102c68f0: str x0, [x26, #32] : ret = bio_iov_iter_get_pages(bio, iter); 0.00 : ffff8000102c68f4: mov x1, x23 0.00 : ffff8000102c68f8: mov x0, x26 : bio->bi_write_hint = iocb->ki_hint; 0.00 : ffff8000102c68fc: ldrh w3, [x20, #36] 0.00 : ffff8000102c6900: strh w3, [x26, #24] : bio->bi_private = dio; 3.81 : ffff8000102c6904: stp x27, x19, [x26, #56] : bio->bi_ioprio = iocb->ki_ioprio; 0.43 : ffff8000102c6908: ldrh w3, [x20, #38] 0.00 : ffff8000102c690c: strh w3, [x26, #22] : ret = bio_iov_iter_get_pages(bio, iter); 0.00 : ffff8000102c6910: bl ffff800010457540 0.00 : ffff8000102c6914: mov w25, w0 : if (unlikely(ret)) { 0.00 : ffff8000102c6918: cbnz w0, ffff8000102c69e4 : if (is_read) { 0.43 : ffff8000102c691c: ldr w0, [x29, #124] 0.00 : ffff8000102c6920: cbz w0, ffff8000102c681c : dio_bio_write_op(): : if (iocb->ki_flags & IOCB_DSYNC) 0.00 : ffff8000102c6924: ldr w0, [x20, #32] : unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; 0.00 : ffff8000102c6928: mov w2, #0x8801 // #34817 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000102c692c: mrs x1, sp_el0 : __blkdev_direct_IO(): : task_io_account_write(bio->bi_iter.bi_size); 0.00 : ffff8000102c6930: ldr w3, [x26, #40] : dio_bio_write_op(): : unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; 0.00 : ffff8000102c6934: tst x0, #0x10 0.00 : ffff8000102c6938: mov w0, #0x8801 // #34817 0.00 : ffff8000102c693c: movk w0, #0x2, lsl #16 0.00 : ffff8000102c6940: csel w0, w0, w2, ne // ne = any : __blkdev_direct_IO(): : bio->bi_opf = dio_bio_write_op(iocb); 0.00 : ffff8000102c6944: str w0, [x26, #16] : task_io_account_write(): : return p->ioac.read_bytes >> 9; : } : : static inline void task_io_account_write(size_t bytes) : { : current->ioac.write_bytes += bytes; 0.00 : ffff8000102c6948: ldr x0, [x1, #1936] 0.00 : ffff8000102c694c: add x0, x0, x3 0.00 : ffff8000102c6950: str x0, [x1, #1936] 0.00 : ffff8000102c6954: b ffff8000102c6828 : __blkdev_direct_IO(): : __set_current_state(TASK_RUNNING); 0.00 : ffff8000102c6958: str xzr, [x0, #24] : if (!ret) 0.00 : ffff8000102c695c: cbz w25, ffff8000102c6ae4 : bio_put(&dio->bio); 0.00 : ffff8000102c6960: mov x0, x21 0.00 : ffff8000102c6964: bl ffff800010456ab8 : return ret; 0.00 : ffff8000102c6968: sxtw x0, w25 : blkdev_direct_IO(): : return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES)); 4.61 : ffff8000102c696c: ldr x19, [x29, #16] 5.92 : ffff8000102c6970: ldp x21, x22, [x29, #32] 0.00 : ffff8000102c6974: ldr x24, [x29, #56] 0.41 : ffff8000102c6978: ldp x26, x27, [x29, #72] 2.98 : ffff8000102c697c: ldr x28, [x29, #88] : } 0.00 : ffff8000102c6980: adrp x1, ffff800011909000 0.00 : ffff8000102c6984: add x25, x1, #0x908 0.00 : ffff8000102c6988: ldr x2, [x29, #168] 0.00 : ffff8000102c698c: ldr x1, [x25] 0.00 : ffff8000102c6990: eor x1, x2, x1 0.00 : ffff8000102c6994: cbnz x1, ffff8000102c6b50 0.00 : ffff8000102c6998: ldr x20, [sp, #24] 0.39 : ffff8000102c699c: ldr x23, [sp, #48] 0.00 : ffff8000102c69a0: ldr x25, [sp, #64] 0.00 : ffff8000102c69a4: ldp x29, x30, [sp], #176 0.00 : ffff8000102c69a8: ret : __ll_sc_atomic_add(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff8000102c69ac: b ffff8000102c827c 0.00 : ffff8000102c69b0: b ffff8000102c6888 : __blkdev_direct_IO(): : atomic_inc(&dio->ref); 0.00 : ffff8000102c69b4: add x1, x19, #0x10 : arch_static_branch_jump(): 0.00 : ffff8000102c69b8: b ffff8000102c69cc 0.00 : ffff8000102c69bc: b ffff8000102c69cc : __lse_atomic_add(): 0.00 : ffff8000102c69c0: mov w0, #0x1 // #1 0.00 : ffff8000102c69c4: stadd w0, [x1] 0.00 : ffff8000102c69c8: b ffff8000102c689c : __ll_sc_atomic_add(): 0.00 : ffff8000102c69cc: add x2, x19, #0x10 0.00 : ffff8000102c69d0: b ffff8000102c8294 0.00 : ffff8000102c69d4: b ffff8000102c689c : __blkdev_direct_IO(): : bio_set_pages_dirty(bio); 1.72 : ffff8000102c69d8: mov x0, x26 0.00 : ffff8000102c69dc: bl ffff800010458448 0.00 : ffff8000102c69e0: b ffff8000102c6828 : bio->bi_status = BLK_STS_IOERR; 0.00 : ffff8000102c69e4: mov w0, #0xa // #10 0.00 : ffff8000102c69e8: strb w0, [x26, #26] : blk_qc_t qc = BLK_QC_T_NONE; 0.00 : ffff8000102c69ec: mov w22, #0xffffffff // #-1 : bio_endio(bio); 0.00 : ffff8000102c69f0: mov x0, x26 0.00 : ffff8000102c69f4: bl ffff800010456b18 : if (!is_poll) 0.00 : ffff8000102c69f8: ldr w0, [x29, #120] 0.00 : ffff8000102c69fc: cbz w0, ffff8000102c6ad8 : if (!is_sync) 0.85 : ffff8000102c6a00: ldr x0, [x29, #104] 0.00 : ffff8000102c6a04: cbnz x0, ffff8000102c6b2c : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000102c6a08: mov x23, #0x2 // #2 0.00 : ffff8000102c6a0c: nop : get_current(): 0.00 : ffff8000102c6a10: mrs x0, sp_el0 : __write_once_size(): 0.00 : ffff8000102c6a14: str x23, [x0, #24] : __blkdev_direct_IO(): : set_current_state(TASK_UNINTERRUPTIBLE); 0.00 : ffff8000102c6a18: dmb ish : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102c6a1c: ldur x1, [x21, #-24] : __blkdev_direct_IO(): : if (!READ_ONCE(dio->waiter)) 0.00 : ffff8000102c6a20: cbz x1, ffff8000102c6958 : if (!(iocb->ki_flags & IOCB_HIPRI) || 0.00 : ffff8000102c6a24: ldr w0, [x20, #32] 0.00 : ffff8000102c6a28: tbz w0, #3, ffff8000102c6a44 : bdev_get_queue(): : return bdev->bd_disk->queue; /* this is never NULL */ 0.00 : ffff8000102c6a2c: ldur x0, [x28, #-88] : __blkdev_direct_IO(): : !blk_poll(bdev_get_queue(bdev), qc, true)) 0.00 : ffff8000102c6a30: mov w2, #0x1 // #1 0.00 : ffff8000102c6a34: mov w1, w22 0.00 : ffff8000102c6a38: ldr x0, [x0, #1040] 0.00 : ffff8000102c6a3c: bl ffff800010464900 : if (!(iocb->ki_flags & IOCB_HIPRI) || 0.00 : ffff8000102c6a40: cbnz w0, ffff8000102c6a10 : io_schedule(); 0.00 : ffff8000102c6a44: bl ffff800010cd3008 0.00 : ffff8000102c6a48: b ffff8000102c6a10 : blkdev_direct_IO(): : return __blkdev_direct_IO_simple(iocb, iter, nr_pages); 0.00 : ffff8000102c6a4c: mov x1, x23 0.00 : ffff8000102c6a50: mov x0, x20 0.00 : ffff8000102c6a54: bl ffff8000102c6140 <__blkdev_direct_IO_simple> 0.00 : ffff8000102c6a58: ldr x24, [x29, #56] 0.00 : ffff8000102c6a5c: b ffff8000102c6980 : __blkdev_direct_IO(): : if (iocb->ki_flags & IOCB_HIPRI) { 1.25 : ffff8000102c6a60: ldr w0, [x20, #32] 0.00 : ffff8000102c6a64: tbz w0, #3, ffff8000102c6b08 : bio_set_polled(): : * must be found by the caller. This is different than IRQ driven IO, where : * it's safe to wait for IO to complete. : */ : static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb) : { : bio->bi_opf |= REQ_HIPRI; 0.00 : ffff8000102c6a68: ldr w0, [x26, #16] 0.00 : ffff8000102c6a6c: orr w1, w0, #0x2000000 0.00 : ffff8000102c6a70: str w1, [x26, #16] : if (!is_sync_kiocb(kiocb)) 0.00 : ffff8000102c6a74: ldr x1, [x20, #16] 0.00 : ffff8000102c6a78: cbz x1, ffff8000102c6b18 : bio->bi_opf |= REQ_NOWAIT; 0.00 : ffff8000102c6a7c: mov w1, #0x2200000 // #35651584 0.00 : ffff8000102c6a80: orr w0, w0, w1 0.00 : ffff8000102c6a84: str w0, [x26, #16] : __blkdev_direct_IO(): : qc = submit_bio(bio); 0.00 : ffff8000102c6a88: mov x0, x26 0.00 : ffff8000102c6a8c: bl ffff80001045b7e0 0.00 : ffff8000102c6a90: mov w22, w0 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102c6a94: str w22, [x20, #40] 0.00 : ffff8000102c6a98: b ffff8000102c69f8 : __blkdev_direct_IO(): : blk_start_plug(&plug); 0.00 : ffff8000102c6a9c: add x0, x29, #0x80 0.43 : ffff8000102c6aa0: bl ffff800010459f98 0.00 : ffff8000102c6aa4: b ffff8000102c67fc : bio_get(): : bio->bi_flags |= (1 << BIO_REFFED); 0.00 : ffff8000102c6aa8: ldrh w0, [x21, #20] : get_current(): 0.00 : ffff8000102c6aac: mrs x1, sp_el0 : bio_get(): 0.00 : ffff8000102c6ab0: orr w0, w0, #0x100 : __blkdev_direct_IO(): : dio->waiter = current; 0.00 : ffff8000102c6ab4: stur x1, [x21, #-24] : bio_get(): 0.00 : ffff8000102c6ab8: strh w0, [x21, #20] : smp_mb__before_atomic(); 0.00 : ffff8000102c6abc: dmb ish : atomic_inc(&bio->__bi_cnt); 0.00 : ffff8000102c6ac0: add x1, x21, #0x64 : arch_static_branch_jump(): 0.00 : ffff8000102c6ac4: b ffff8000102c6afc 0.00 : ffff8000102c6ac8: b ffff8000102c6afc : __lse_atomic_add(): 0.00 : ffff8000102c6acc: mov w0, #0x1 // #1 0.00 : ffff8000102c6ad0: stadd w0, [x1] 0.00 : ffff8000102c6ad4: b ffff8000102c67c0 : __blkdev_direct_IO(): : blk_finish_plug(&plug); 1.28 : ffff8000102c6ad8: add x0, x29, #0x80 0.00 : ffff8000102c6adc: bl ffff80001045cd08 1.67 : ffff8000102c6ae0: b ffff8000102c6a00 : ret = blk_status_to_errno(dio->bio.bi_status); 0.00 : ffff8000102c6ae4: ldrb w0, [x19, #50] 0.00 : ffff8000102c6ae8: bl ffff800010459fd0 0.00 : ffff8000102c6aec: mov w25, w0 : if (likely(!ret)) 0.00 : ffff8000102c6af0: cbnz w0, ffff8000102c6960 : ret = dio->size; 0.00 : ffff8000102c6af4: ldur w25, [x21, #-16] 0.00 : ffff8000102c6af8: b ffff8000102c6960 : __ll_sc_atomic_add(): 0.00 : ffff8000102c6afc: add x2, x21, #0x64 0.00 : ffff8000102c6b00: b ffff8000102c82ac 0.00 : ffff8000102c6b04: b ffff8000102c67c0 : __blkdev_direct_IO(): : qc = submit_bio(bio); 4.27 : ffff8000102c6b08: mov x0, x26 0.00 : ffff8000102c6b0c: bl ffff80001045b7e0 0.00 : ffff8000102c6b10: mov w22, w0 0.00 : ffff8000102c6b14: b ffff8000102c69f8 0.00 : ffff8000102c6b18: mov x0, x26 0.00 : ffff8000102c6b1c: bl ffff80001045b7e0 0.00 : ffff8000102c6b20: mov w22, w0 : __write_once_size(): 0.00 : ffff8000102c6b24: str w22, [x20, #40] 0.00 : ffff8000102c6b28: b ffff8000102c69f8 : __blkdev_direct_IO(): : return -EIOCBQUEUED; 2.55 : ffff8000102c6b2c: mov x0, #0xfffffffffffffdef // #-529 0.00 : ffff8000102c6b30: b ffff8000102c696c : return -EINVAL; 0.00 : ffff8000102c6b34: mov x0, #0xffffffffffffffea // #-22 0.00 : ffff8000102c6b38: ldr x19, [x29, #16] 0.00 : ffff8000102c6b3c: ldr x22, [x29, #40] 0.00 : ffff8000102c6b40: ldr x24, [x29, #56] 0.00 : ffff8000102c6b44: ldr x26, [x29, #72] 0.00 : ffff8000102c6b48: ldr x28, [x29, #88] 0.00 : ffff8000102c6b4c: b ffff8000102c6980 0.00 : ffff8000102c6b50: str x19, [x29, #16] 0.00 : ffff8000102c6b54: stp x21, x22, [x29, #32] 0.00 : ffff8000102c6b58: str x24, [x29, #56] 0.00 : ffff8000102c6b5c: stp x26, x27, [x29, #72] 0.00 : ffff8000102c6b60: str x28, [x29, #88] : blkdev_direct_IO(): : } 0.00 : ffff8000102c6b64: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (159 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107d47a8 : : nvme_queue_rq(): : /* : * NOTE: ns is NULL when called on the admin queue. : */ : static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, : const struct blk_mq_queue_data *bd) : { 1.85 : ffff8000107d47a8: stp x29, x30, [sp, #-256]! 0.00 : ffff8000107d47ac: mov x29, sp 0.00 : ffff8000107d47b0: stp x21, x22, [sp, #32] 0.00 : ffff8000107d47b4: mov x21, x1 0.64 : ffff8000107d47b8: stp x19, x20, [sp, #16] 0.00 : ffff8000107d47bc: adrp x19, ffff800011909000 2.49 : ffff8000107d47c0: str x23, [sp, #48] 0.00 : ffff8000107d47c4: add x1, x19, #0x908 8.17 : ffff8000107d47c8: str x28, [sp, #88] : struct nvme_ns *ns = hctx->queue->queuedata; : struct nvme_queue *nvmeq = hctx->driver_data; : struct nvme_dev *dev = nvmeq->dev; : struct request *req = bd->rq; 3.18 : ffff8000107d47cc: ldr x28, [x21] : { 0.63 : ffff8000107d47d0: ldr x2, [x1] 0.00 : ffff8000107d47d4: str x2, [x29, #248] 0.00 : ffff8000107d47d8: mov x2, #0x0 // #0 : struct nvme_ns *ns = hctx->queue->queuedata; 1.91 : ffff8000107d47dc: ldr x3, [x0, #208] : struct nvme_iod *iod = blk_mq_rq_to_pdu(req); : struct nvme_command cmnd; : blk_status_t ret; : : iod->aborted = 0; 0.00 : ffff8000107d47e0: mov x2, #0xffffffff00000000 // #-4294967296 : struct nvme_queue *nvmeq = hctx->driver_data; 1.29 : ffff8000107d47e4: ldr x20, [x0, #224] : iod->aborted = 0; 0.00 : ffff8000107d47e8: add x1, x28, #0x200 : struct nvme_ns *ns = hctx->queue->queuedata; 1.27 : ffff8000107d47ec: ldr x0, [x3, #96] : struct nvme_dev *dev = nvmeq->dev; 3.10 : ffff8000107d47f0: ldr x22, [x20] : iod->aborted = 0; 9.95 : ffff8000107d47f4: stur x2, [x1, #-188] : iod->npages = -1; 0.00 : ffff8000107d47f8: str wzr, [x28, #332] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000107d47fc: ldr x1, [x20, #120] : nvme_queue_rq(): : : /* : * We should not need to do this, but we're still using this to : * ensure we can drain requests on a dying queue. : */ : if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags))) 5.01 : ffff8000107d4800: tbz w1, #0, ffff8000107d4c94 0.00 : ffff8000107d4804: str x25, [x29, #64] : return BLK_STS_IOERR; : : ret = nvme_setup_cmd(ns, req, &cmnd); 0.00 : ffff8000107d4808: add x2, x29, #0xb8 0.00 : ffff8000107d480c: and w25, w1, #0x1 0.00 : ffff8000107d4810: mov x1, x28 1.27 : ffff8000107d4814: bl ffff8000107d0750 : if (ret) 4.49 : ffff8000107d4818: ands w23, w0, #0xff 0.00 : ffff8000107d481c: b.ne ffff8000107d4a74 // b.any 0.00 : ffff8000107d4820: str x24, [x29, #56] 3.05 : ffff8000107d4824: stp x26, x27, [x29, #72] : blk_rq_nr_phys_segments(): : * own special payload. In that case we still return 1 here so that this : * special payload will be mapped. : */ : static inline unsigned short blk_rq_nr_phys_segments(struct request *rq) : { : if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 0.00 : ffff8000107d4828: ldr w0, [x28, #28] 0.00 : ffff8000107d482c: tbnz w0, #18, ffff8000107d4c9c : return 1; : return rq->nr_phys_segments; 0.00 : ffff8000107d4830: ldrh w0, [x28, #194] : nvme_queue_rq(): : return ret; : : if (blk_rq_nr_phys_segments(req)) { 0.00 : ffff8000107d4834: cbz w0, ffff8000107d4b18 0.00 : ffff8000107d4838: add x24, x28, #0x118 : nvme_map_data(): : if (blk_rq_nr_phys_segments(req) == 1) { 0.00 : ffff8000107d483c: cmp w0, #0x1 0.00 : ffff8000107d4840: b.eq ffff8000107d4bac // b.none : iod->dma_len = 0; 0.00 : ffff8000107d4844: str wzr, [x24, #64] : iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC); 0.00 : ffff8000107d4848: mov w1, #0xa20 // #2592 0.00 : ffff8000107d484c: ldr x0, [x22, #3176] 0.00 : ffff8000107d4850: bl ffff8000101d7640 0.00 : ffff8000107d4854: str x0, [x24, #80] : if (!iod->sg) 0.00 : ffff8000107d4858: cbz x0, ffff8000107d4d2c : blk_rq_nr_phys_segments(): : if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 0.00 : ffff8000107d485c: ldr w2, [x28, #28] 0.00 : ffff8000107d4860: mov w1, #0x1 // #1 0.00 : ffff8000107d4864: tbnz w2, #18, ffff8000107d486c 0.00 : ffff8000107d4868: ldrh w1, [x28, #194] : nvme_map_data(): : sg_init_table(iod->sg, blk_rq_nr_phys_segments(req)); 0.00 : ffff8000107d486c: bl ffff800010486940 : iod->nents = blk_rq_map_sg(req->q, req, iod->sg); 0.00 : ffff8000107d4870: ldr x2, [x24, #80] 0.00 : ffff8000107d4874: mov x1, x28 0.00 : ffff8000107d4878: ldr x0, [x28] 0.00 : ffff8000107d487c: bl ffff800010460dd0 0.00 : ffff8000107d4880: str w0, [x24, #52] 0.00 : ffff8000107d4884: mov w2, w0 : if (!iod->nents) 0.00 : ffff8000107d4888: cbz w0, ffff8000107d4c60 : nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, 0.00 : ffff8000107d488c: ldr x0, [x22, #336] : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 0.00 : ffff8000107d4890: ldr w3, [x28, #24] : nvme_map_data(): : if (is_pci_p2pdma_page(sg_page(iod->sg))) 0.00 : ffff8000107d4894: ldr x1, [x24, #80] : dma_map_sg_attrs(): : */ : static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, : int nents, enum dma_data_direction dir, : unsigned long attrs) : { : const struct dma_map_ops *ops = get_dma_ops(dev); 0.00 : ffff8000107d4898: ldr x4, [x0, #576] : nvme_map_data(): : nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, 0.00 : ffff8000107d489c: tst x3, #0x1 0.00 : ffff8000107d48a0: cset w3, eq // eq = none 0.00 : ffff8000107d48a4: add w3, w3, #0x1 : get_dma_ops(): : if (dev->dma_ops) 0.00 : ffff8000107d48a8: cbz x4, ffff8000107d4c84 : dma_map_sg_attrs(): : : BUG_ON(!valid_dma_direction(dir)); : if (dma_is_direct(ops)) : ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); : else : ents = ops->map_sg(dev, sg, nents, dir, attrs); 0.00 : ffff8000107d48ac: ldr x5, [x4, #48] 0.00 : ffff8000107d48b0: mov x4, #0x100 // #256 0.00 : ffff8000107d48b4: blr x5 0.00 : ffff8000107d48b8: mov w27, w0 : BUG_ON(ents < 0); 0.00 : ffff8000107d48bc: cmp w27, #0x0 0.00 : ffff8000107d48c0: b.lt ffff8000107d4dd0 // b.tstop : nvme_map_data(): : if (!nr_mapped) 0.00 : ffff8000107d48c4: b.eq ffff8000107d4c60 // b.none : blk_rq_nr_phys_segments(): 0.00 : ffff8000107d48c8: ldr w0, [x28, #28] 0.00 : ffff8000107d48cc: and w7, w0, #0x40000 0.00 : ffff8000107d48d0: tbz w0, #18, ffff8000107d4aa8 : blk_rq_payload_bytes(): : return rq->special_vec.bv_len; 0.00 : ffff8000107d48d4: ldr w0, [x28, #112] 0.00 : ffff8000107d48d8: mov w2, #0x1 // #1 : blk_rq_nr_phys_segments(): : return 1; 0.00 : ffff8000107d48dc: mov w1, w2 : nvme_pci_use_sgls(): : if (!(dev->ctrl.sgls & ((1 << 0) | (1 << 1)))) 0.00 : ffff8000107d48e0: ldr w5, [x22, #1764] 0.00 : ffff8000107d48e4: ldr x26, [x24, #80] 0.00 : ffff8000107d48e8: tst x5, #0x3 0.00 : ffff8000107d48ec: b.eq ffff8000107d4d34 // b.none : if (!iod->nvmeq->qid) 0.00 : ffff8000107d48f0: ldr x5, [x24, #32] 0.00 : ffff8000107d48f4: ldrh w5, [x5, #114] 0.00 : ffff8000107d48f8: cbz w5, ffff8000107d4d34 : if (!sgl_threshold || avg_seg_size < sgl_threshold) 0.00 : ffff8000107d48fc: adrp x5, ffff800011a93000 0.00 : ffff8000107d4900: ldr w5, [x5, #2088] 0.00 : ffff8000107d4904: cbz w5, ffff8000107d4d34 : avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg); 0.00 : ffff8000107d4908: add w0, w2, w0 0.00 : ffff8000107d490c: sub w0, w0, #0x1 0.00 : ffff8000107d4910: udiv w0, w0, w2 : if (!sgl_threshold || avg_seg_size < sgl_threshold) 0.00 : ffff8000107d4914: cmp w0, w5 0.00 : ffff8000107d4918: b.cc ffff8000107d4d34 // b.lo, b.ul, b.last : nvme_map_data(): : iod->use_sgl = nvme_pci_use_sgls(dev, req); 0.00 : ffff8000107d491c: mov w1, #0x1 // #1 : nvme_pci_setup_sgls(): : cmd->flags = NVME_CMD_SGL_METABUF; 0.00 : ffff8000107d4920: mov w0, #0x40 // #64 : nvme_map_data(): : iod->use_sgl = nvme_pci_use_sgls(dev, req); 0.00 : ffff8000107d4924: strb w1, [x24, #40] : nvme_pci_setup_sgls(): : if (entries == 1) { 0.00 : ffff8000107d4928: cmp w27, #0x1 : cmd->flags = NVME_CMD_SGL_METABUF; 0.00 : ffff8000107d492c: strb w0, [x29, #185] : if (entries == 1) { 0.00 : ffff8000107d4930: b.eq ffff8000107d4f38 // b.none : if (entries <= (256 / sizeof(struct nvme_sgl_desc))) { 0.00 : ffff8000107d4934: cmp w27, #0x10 0.00 : ffff8000107d4938: b.gt ffff8000107d4de0 : pool = dev->prp_small_pool; 0.00 : ffff8000107d493c: ldr x0, [x22, #352] 0.00 : ffff8000107d4940: str x0, [x29, #136] : iod->npages = 0; 0.00 : ffff8000107d4944: str wzr, [x24, #48] : sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma); 0.00 : ffff8000107d4948: ldr x0, [x29, #136] 0.00 : ffff8000107d494c: mov w1, #0xa20 // #2592 0.00 : ffff8000107d4950: add x2, x29, #0xb0 0.00 : ffff8000107d4954: bl ffff800010239bc8 0.00 : ffff8000107d4958: mov x5, x0 : if (!sg_list) { 0.00 : ffff8000107d495c: cbz x0, ffff8000107d4fd0 : blk_rq_nr_phys_segments(): : if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 0.00 : ffff8000107d4960: ldr w2, [x28, #28] 0.00 : ffff8000107d4964: mov x0, #0x20 // #32 : nvme_pci_iod_list(): : return (void **)(iod->sg + blk_rq_nr_phys_segments(req)); 0.00 : ffff8000107d4968: ldr x1, [x24, #80] : blk_rq_nr_phys_segments(): 0.00 : ffff8000107d496c: tbnz w2, #18, ffff8000107d4978 0.00 : ffff8000107d4970: ldrh w0, [x28, #194] 0.00 : ffff8000107d4974: lsl x0, x0, #5 : nvme_pci_setup_sgls(): : nvme_pci_iod_list(req)[0] = sg_list; 0.00 : ffff8000107d4978: str x5, [x1, x0] : nvme_pci_sgl_set_seg(): : if (entries < SGES_PER_PAGE) { 0.00 : ffff8000107d497c: cmp w27, #0xff : nvme_pci_setup_sgls(): : iod->first_dma = sgl_dma; 0.00 : ffff8000107d4980: ldr x0, [x29, #176] 0.00 : ffff8000107d4984: str x0, [x24, #56] : nvme_pci_sgl_set_seg(): : sge->addr = cpu_to_le64(dma_addr); 0.00 : ffff8000107d4988: str x0, [x29, #208] : if (entries < SGES_PER_PAGE) { 0.00 : ffff8000107d498c: b.gt ffff8000107d4f50 : sge->length = cpu_to_le32(entries * sizeof(*sge)); 0.00 : ffff8000107d4990: lsl w1, w27, #4 : sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4; 0.00 : ffff8000107d4994: mov w0, #0x30 // #48 : sge->length = cpu_to_le32(entries * sizeof(*sge)); 0.00 : ffff8000107d4998: str w1, [x29, #216] : sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4; 0.00 : ffff8000107d499c: strb w0, [x29, #223] : sge->type = NVME_SGL_FMT_SEG_DESC << 4; 0.00 : ffff8000107d49a0: mov w1, #0x0 // #0 0.00 : ffff8000107d49a4: nop : nvme_pci_setup_sgls(): : nvme_pci_sgl_set_data(&sg_list[i++], sg); 0.00 : ffff8000107d49a8: sbfiz x0, x1, #4, #32 0.00 : ffff8000107d49ac: ldr x7, [x26, #16] 0.00 : ffff8000107d49b0: add x2, x5, x0 0.00 : ffff8000107d49b4: add w1, w1, #0x1 0.00 : ffff8000107d49b8: str w1, [x29, #148] : nvme_pci_sgl_set_data(): : sge->addr = cpu_to_le64(sg_dma_address(sg)); 0.00 : ffff8000107d49bc: str x7, [x5, x0] : nvme_pci_setup_sgls(): : sg = sg_next(sg); 0.00 : ffff8000107d49c0: mov x0, x26 : nvme_pci_sgl_set_data(): : sge->addr = cpu_to_le64(sg_dma_address(sg)); 0.00 : ffff8000107d49c4: str x5, [x29, #152] : sge->length = cpu_to_le32(sg_dma_len(sg)); 0.00 : ffff8000107d49c8: ldr w4, [x26, #24] 0.00 : ffff8000107d49cc: str w4, [x2, #8] : sge->type = NVME_SGL_FMT_DATA_DESC << 4; 0.00 : ffff8000107d49d0: strb wzr, [x2, #15] : nvme_pci_setup_sgls(): : sg = sg_next(sg); 0.00 : ffff8000107d49d4: bl ffff8000104867f8 0.00 : ffff8000107d49d8: mov x26, x0 : } while (--entries > 0); 0.00 : ffff8000107d49dc: subs w27, w27, #0x1 0.00 : ffff8000107d49e0: ldr w1, [x29, #148] 0.00 : ffff8000107d49e4: ldr x5, [x29, #152] 0.00 : ffff8000107d49e8: b.eq ffff8000107d4b18 // b.none : if (i == SGES_PER_PAGE) { 0.00 : ffff8000107d49ec: cmp w1, #0x100 0.00 : ffff8000107d49f0: b.ne ffff8000107d49a8 // b.any : sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma); 0.00 : ffff8000107d49f4: ldr x0, [x29, #136] 0.00 : ffff8000107d49f8: add x2, x29, #0xb0 0.00 : ffff8000107d49fc: mov w1, #0xa20 // #2592 0.00 : ffff8000107d4a00: str x5, [x29, #152] 0.00 : ffff8000107d4a04: bl ffff800010239bc8 : if (!sg_list) 0.00 : ffff8000107d4a08: cbz x0, ffff8000107d4f94 : blk_rq_nr_phys_segments(): 0.00 : ffff8000107d4a0c: ldr w2, [x28, #28] 0.00 : ffff8000107d4a10: mov x1, #0x20 // #32 : nvme_pci_iod_list(): : return (void **)(iod->sg + blk_rq_nr_phys_segments(req)); 0.00 : ffff8000107d4a14: ldr x7, [x24, #80] : blk_rq_nr_phys_segments(): 0.00 : ffff8000107d4a18: ldr x5, [x29, #152] 0.00 : ffff8000107d4a1c: tbnz w2, #18, ffff8000107d4a28 0.00 : ffff8000107d4a20: ldrh w1, [x28, #194] 0.00 : ffff8000107d4a24: lsl x1, x1, #5 : nvme_pci_setup_sgls(): : nvme_pci_iod_list(req)[iod->npages++] = sg_list; 0.00 : ffff8000107d4a28: ldr w2, [x24, #48] : nvme_pci_sgl_set_seg(): : if (entries < SGES_PER_PAGE) { 0.00 : ffff8000107d4a2c: cmp w27, #0xff : nvme_pci_setup_sgls(): : nvme_pci_iod_list(req)[iod->npages++] = sg_list; 0.00 : ffff8000107d4a30: add x1, x1, w2, sxtw #3 0.00 : ffff8000107d4a34: add w2, w2, #0x1 0.00 : ffff8000107d4a38: str w2, [x24, #48] : sg_list[i++] = *link; 0.00 : ffff8000107d4a3c: add x2, x5, #0xff0 : nvme_pci_iod_list(req)[iod->npages++] = sg_list; 0.00 : ffff8000107d4a40: str x0, [x7, x1] : sg_list[i++] = *link; 0.00 : ffff8000107d4a44: ldp x8, x9, [x2] 0.00 : ffff8000107d4a48: stp x8, x9, [x0] : nvme_pci_sgl_set_seg(): : sge->addr = cpu_to_le64(dma_addr); 0.00 : ffff8000107d4a4c: ldr x1, [x29, #176] 0.00 : ffff8000107d4a50: str x1, [x5, #4080] : if (entries < SGES_PER_PAGE) { 0.00 : ffff8000107d4a54: b.gt ffff8000107d4d5c : sge->length = cpu_to_le32(entries * sizeof(*sge)); 0.00 : ffff8000107d4a58: lsl w5, w27, #4 : sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4; 0.00 : ffff8000107d4a5c: mov w1, #0x30 // #48 : sge->length = cpu_to_le32(entries * sizeof(*sge)); 0.00 : ffff8000107d4a60: str w5, [x2, #8] : sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4; 0.00 : ffff8000107d4a64: mov x5, x0 0.00 : ffff8000107d4a68: strb w1, [x2, #15] : nvme_pci_setup_sgls(): : sg_list[i++] = *link; 0.00 : ffff8000107d4a6c: mov w1, w25 0.00 : ffff8000107d4a70: b ffff8000107d49a8 0.00 : ffff8000107d4a74: ldr x25, [x29, #64] : nvme_queue_rq(): : out_unmap_data: : nvme_unmap_data(dev, req); : out_free_cmd: : nvme_cleanup_cmd(req); : return ret; : } 0.00 : ffff8000107d4a78: add x19, x19, #0x908 0.00 : ffff8000107d4a7c: mov w0, w23 0.00 : ffff8000107d4a80: ldr x2, [x29, #248] 3.14 : ffff8000107d4a84: ldr x1, [x19] 0.00 : ffff8000107d4a88: eor x1, x2, x1 0.00 : ffff8000107d4a8c: cbnz x1, ffff8000107d5060 0.63 : ffff8000107d4a90: ldp x19, x20, [sp, #16] 0.62 : ffff8000107d4a94: ldp x21, x22, [sp, #32] 0.00 : ffff8000107d4a98: ldr x23, [sp, #48] 0.60 : ffff8000107d4a9c: ldr x28, [sp, #88] 0.00 : ffff8000107d4aa0: ldp x29, x30, [sp], #256 0.00 : ffff8000107d4aa4: ret : blk_rq_nr_phys_segments(): : return rq->nr_phys_segments; 0.00 : ffff8000107d4aa8: ldrh w1, [x28, #194] 0.00 : ffff8000107d4aac: ldr w0, [x28, #40] : nvme_pci_use_sgls(): : if (nseg == 0) 0.00 : ffff8000107d4ab0: cbnz w1, ffff8000107d4dc8 : nvme_map_data(): : iod->use_sgl = nvme_pci_use_sgls(dev, req); 0.00 : ffff8000107d4ab4: strb wzr, [x24, #40] 0.00 : ffff8000107d4ab8: mov x4, #0x0 // #0 : nvme_pci_setup_prps(): : struct scatterlist *sg = iod->sg; 0.00 : ffff8000107d4abc: ldr x26, [x24, #80] : u32 page_size = dev->ctrl.page_size; 0.00 : ffff8000107d4ac0: ldr w7, [x22, #1720] : u64 dma_addr = sg_dma_address(sg); 0.00 : ffff8000107d4ac4: ldr x5, [x26, #16] : int offset = dma_addr & (page_size - 1); 0.00 : ffff8000107d4ac8: sub w27, w7, #0x1 : int dma_len = sg_dma_len(sg); 0.00 : ffff8000107d4acc: ldr w2, [x26, #24] : int offset = dma_addr & (page_size - 1); 0.00 : ffff8000107d4ad0: and w1, w27, w5 : length -= (page_size - offset); 0.00 : ffff8000107d4ad4: sub w3, w1, w7 0.00 : ffff8000107d4ad8: add w25, w3, w0 0.00 : ffff8000107d4adc: mov w8, w25 : if (length <= 0) { 0.00 : ffff8000107d4ae0: cmp w25, #0x0 0.00 : ffff8000107d4ae4: b.le ffff8000107d4dd4 : if (dma_len) { 0.00 : ffff8000107d4ae8: adds w3, w3, w2 0.00 : ffff8000107d4aec: b.eq ffff8000107d4f64 // b.none : dma_addr += (page_size - offset); 0.00 : ffff8000107d4af0: sub w1, w7, w1 0.00 : ffff8000107d4af4: mov x9, x26 0.00 : ffff8000107d4af8: add x5, x1, x5 : if (length <= page_size) { 0.00 : ffff8000107d4afc: cmp w25, w7 0.00 : ffff8000107d4b00: b.hi ffff8000107d4df4 // b.pmore 0.00 : ffff8000107d4b04: ldr x26, [x24, #80] : iod->first_dma = dma_addr; 0.00 : ffff8000107d4b08: str x5, [x24, #56] : cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); 0.00 : ffff8000107d4b0c: ldr x0, [x26, #16] : cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma); 0.00 : ffff8000107d4b10: stp x0, x5, [x29, #208] 0.00 : ffff8000107d4b14: nop : nvme_queue_rq(): : if (blk_integrity_rq(req)) { 1.86 : ffff8000107d4b18: ldr w3, [x28, #24] 0.00 : ffff8000107d4b1c: tbz w3, #16, ffff8000107d4b7c : rq_integrity_vec(): : * Return the first bvec that contains integrity data. Only drivers that are : * limited to a single integrity segment should use this helper. : */ : static inline struct bio_vec *rq_integrity_vec(struct request *rq) : { : if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1)) 0.00 : ffff8000107d4b20: ldr x1, [x28] : nvme_queue_rq(): : ret = nvme_map_metadata(dev, req, &cmnd); 0.00 : ffff8000107d4b24: ldr x0, [x22, #336] : rq_integrity_vec(): 0.00 : ffff8000107d4b28: ldrh w1, [x1, #1130] 0.00 : ffff8000107d4b2c: cmp w1, #0x1 0.00 : ffff8000107d4b30: b.hi ffff8000107d4d84 // b.pmore : return NULL; : return rq->bio->bi_integrity->bip_vec; 0.00 : ffff8000107d4b34: ldr x1, [x28, #56] 0.00 : ffff8000107d4b38: ldr x1, [x1, #88] 0.00 : ffff8000107d4b3c: ldr x5, [x1, #96] : nvme_map_metadata(): : iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req), 0.00 : ffff8000107d4b40: ldr x1, [x5] 0.00 : ffff8000107d4b44: ldr w2, [x5, #12] : dma_map_page_attrs(): : const struct dma_map_ops *ops = get_dma_ops(dev); 0.00 : ffff8000107d4b48: ldr x7, [x0, #576] : nvme_map_metadata(): 0.00 : ffff8000107d4b4c: tst x3, #0x1 0.00 : ffff8000107d4b50: cset w4, eq // eq = none 0.00 : ffff8000107d4b54: ldr w3, [x5, #8] 0.00 : ffff8000107d4b58: add w4, w4, #0x1 : get_dma_ops(): : if (dev->dma_ops) 0.00 : ffff8000107d4b5c: cbz x7, ffff8000107d4ba0 : dma_map_page_attrs(): : addr = ops->map_page(dev, page, offset, size, dir, attrs); 0.00 : ffff8000107d4b60: ldr x7, [x7, #32] 0.00 : ffff8000107d4b64: mov x5, #0x0 // #0 0.00 : ffff8000107d4b68: blr x7 : nvme_map_metadata(): 0.00 : ffff8000107d4b6c: str x0, [x28, #352] : dma_mapping_error(): : : static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) : { : debug_dma_mapping_error(dev, dma_addr); : : if (dma_addr == DMA_MAPPING_ERROR) 0.00 : ffff8000107d4b70: cmn x0, #0x1 0.00 : ffff8000107d4b74: b.eq ffff8000107d4d18 // b.none : nvme_map_metadata(): : cmnd->rw.metadata = cpu_to_le64(iod->meta_dma); 0.00 : ffff8000107d4b78: str x0, [x29, #200] : nvme_queue_rq(): : blk_mq_start_request(req); 1.81 : ffff8000107d4b7c: mov x0, x28 0.00 : ffff8000107d4b80: bl ffff800010464450 : nvme_submit_cmd(nvmeq, &cmnd, bd->last); 1.28 : ffff8000107d4b84: ldrb w2, [x21, #8] 0.00 : ffff8000107d4b88: add x1, x29, #0xb8 0.00 : ffff8000107d4b8c: mov x0, x20 0.00 : ffff8000107d4b90: bl ffff8000107d30a0 : return BLK_STS_OK; 0.00 : ffff8000107d4b94: ldp x24, x25, [x29, #56] 1.92 : ffff8000107d4b98: ldp x26, x27, [x29, #72] 0.00 : ffff8000107d4b9c: b ffff8000107d4a78 : dma_map_page_attrs(): : addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); 0.00 : ffff8000107d4ba0: mov x5, #0x0 // #0 0.00 : ffff8000107d4ba4: bl ffff800010163c08 0.00 : ffff8000107d4ba8: b ffff8000107d4b6c : req_bvec(): : return mp_bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter); 0.00 : ffff8000107d4bac: ldr x1, [x28, #56] 0.64 : ffff8000107d4bb0: ldp w2, w0, [x1, #40] 9.51 : ffff8000107d4bb4: ldr x4, [x1, #104] 2.54 : ffff8000107d4bb8: ldr w27, [x1, #48] 0.00 : ffff8000107d4bbc: lsl x0, x0, #4 0.00 : ffff8000107d4bc0: add x3, x4, x0 0.00 : ffff8000107d4bc4: ldr x1, [x4, x0] 2.53 : ffff8000107d4bc8: ldp w26, w0, [x3, #8] 0.00 : ffff8000107d4bcc: sub w26, w26, w27 0.00 : ffff8000107d4bd0: add w27, w27, w0 0.00 : ffff8000107d4bd4: cmp w26, w2 6.29 : ffff8000107d4bd8: csel w26, w26, w2, ls // ls = plast : nvme_map_data(): : if (bv.bv_offset + bv.bv_len <= dev->ctrl.page_size * 2) 0.63 : ffff8000107d4bdc: ldr w24, [x22, #1720] 0.00 : ffff8000107d4be0: add w0, w27, w26 0.00 : ffff8000107d4be4: cmp w0, w24, lsl #1 0.00 : ffff8000107d4be8: b.ls ffff8000107d4ca8 // b.plast : if (iod->nvmeq->qid && 0.00 : ffff8000107d4bec: add x24, x28, #0x118 0.00 : ffff8000107d4bf0: ldr x0, [x24, #32] 0.00 : ffff8000107d4bf4: ldrh w0, [x0, #114] 0.00 : ffff8000107d4bf8: cbz w0, ffff8000107d4844 : dev->ctrl.sgls & ((1 << 0) | (1 << 1))) 0.00 : ffff8000107d4bfc: ldr w0, [x22, #1764] : if (iod->nvmeq->qid && 0.00 : ffff8000107d4c00: tst x0, #0x3 0.00 : ffff8000107d4c04: b.eq ffff8000107d4844 // b.none : return nvme_setup_sgl_simple(dev, req, 0.00 : ffff8000107d4c08: ldr x0, [x22, #336] : nvme_setup_sgl_simple(): : iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0); 0.00 : ffff8000107d4c0c: mov w2, w27 : op_is_write(): 0.00 : ffff8000107d4c10: ldr w4, [x28, #24] : nvme_setup_sgl_simple(): 0.00 : ffff8000107d4c14: mov w3, w26 : dma_map_page_attrs(): : const struct dma_map_ops *ops = get_dma_ops(dev); 0.00 : ffff8000107d4c18: ldr x5, [x0, #576] : nvme_setup_sgl_simple(): 0.00 : ffff8000107d4c1c: tst x4, #0x1 0.00 : ffff8000107d4c20: cset w4, eq // eq = none 0.00 : ffff8000107d4c24: add w4, w4, #0x1 : get_dma_ops(): : if (dev->dma_ops) 0.00 : ffff8000107d4c28: cbz x5, ffff8000107d4d78 : dma_map_page_attrs(): : addr = ops->map_page(dev, page, offset, size, dir, attrs); 0.00 : ffff8000107d4c2c: ldr x7, [x5, #32] 0.00 : ffff8000107d4c30: mov x5, #0x0 // #0 0.00 : ffff8000107d4c34: blr x7 : nvme_setup_sgl_simple(): 0.00 : ffff8000107d4c38: str x0, [x24, #56] : dma_mapping_error(): : if (dma_addr == DMA_MAPPING_ERROR) 0.00 : ffff8000107d4c3c: cmn x0, #0x1 0.00 : ffff8000107d4c40: b.eq ffff8000107d4d2c // b.none : nvme_setup_sgl_simple(): : cmnd->flags = NVME_CMD_SGL_METABUF; 0.00 : ffff8000107d4c44: mov w1, #0x40 // #64 : cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4; 0.00 : ffff8000107d4c48: strb wzr, [x29, #223] : cmnd->flags = NVME_CMD_SGL_METABUF; 0.00 : ffff8000107d4c4c: strb w1, [x29, #185] : iod->dma_len = bv->bv_len; 0.00 : ffff8000107d4c50: str w26, [x24, #64] : cmnd->dptr.sgl.addr = cpu_to_le64(iod->first_dma); 0.00 : ffff8000107d4c54: str x0, [x29, #208] : cmnd->dptr.sgl.length = cpu_to_le32(iod->dma_len); 0.00 : ffff8000107d4c58: str w26, [x29, #216] 0.00 : ffff8000107d4c5c: b ffff8000107d4b18 : nvme_map_data(): : nvme_unmap_data(dev, req); 0.00 : ffff8000107d4c60: mov x1, x28 0.00 : ffff8000107d4c64: mov x0, x22 : blk_status_t ret = BLK_STS_RESOURCE; 0.00 : ffff8000107d4c68: mov w23, #0x9 // #9 : nvme_unmap_data(dev, req); 0.00 : ffff8000107d4c6c: bl ffff8000107d35e0 : nvme_queue_rq(): : nvme_cleanup_cmd(req); 0.00 : ffff8000107d4c70: mov x0, x28 0.00 : ffff8000107d4c74: bl ffff8000107cfe50 : return ret; 0.00 : ffff8000107d4c78: ldp x24, x25, [x29, #56] 0.00 : ffff8000107d4c7c: ldp x26, x27, [x29, #72] 0.00 : ffff8000107d4c80: b ffff8000107d4a78 : dma_map_sg_attrs(): : ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); 0.00 : ffff8000107d4c84: mov x4, #0x100 // #256 0.00 : ffff8000107d4c88: bl ffff800010163da0 0.00 : ffff8000107d4c8c: mov w27, w0 0.00 : ffff8000107d4c90: b ffff8000107d48bc : nvme_queue_rq(): : return BLK_STS_IOERR; 0.00 : ffff8000107d4c94: mov w23, #0xa // #10 0.00 : ffff8000107d4c98: b ffff8000107d4a78 : req_bvec(): : return rq->special_vec; 0.00 : ffff8000107d4c9c: ldp w26, w27, [x28, #112] 0.00 : ffff8000107d4ca0: ldr x1, [x28, #104] 0.00 : ffff8000107d4ca4: b ffff8000107d4bdc : nvme_map_data(): : return nvme_setup_prp_simple(dev, req, 2.52 : ffff8000107d4ca8: ldr x0, [x22, #336] : nvme_setup_prp_simple(): : iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0); 0.00 : ffff8000107d4cac: mov w2, w27 : op_is_write(): 5.73 : ffff8000107d4cb0: ldr w4, [x28, #24] : nvme_setup_prp_simple(): 0.00 : ffff8000107d4cb4: mov w3, w26 : dma_map_page_attrs(): : const struct dma_map_ops *ops = get_dma_ops(dev); 0.00 : ffff8000107d4cb8: ldr x5, [x0, #576] : nvme_setup_prp_simple(): 0.00 : ffff8000107d4cbc: tst x4, #0x1 1.88 : ffff8000107d4cc0: cset w4, eq // eq = none 0.00 : ffff8000107d4cc4: add w4, w4, #0x1 : get_dma_ops(): : if (dev->dma_ops) 6.32 : ffff8000107d4cc8: cbz x5, ffff8000107d4d0c : dma_map_page_attrs(): : addr = ops->map_page(dev, page, offset, size, dir, attrs); 0.00 : ffff8000107d4ccc: ldr x7, [x5, #32] 0.00 : ffff8000107d4cd0: mov x5, #0x0 // #0 0.00 : ffff8000107d4cd4: blr x7 : nvme_setup_prp_simple(): 0.63 : ffff8000107d4cd8: str x0, [x28, #336] : dma_mapping_error(): : if (dma_addr == DMA_MAPPING_ERROR) 0.00 : ffff8000107d4cdc: cmn x0, #0x1 0.00 : ffff8000107d4ce0: b.eq ffff8000107d4d2c // b.none : nvme_setup_prp_simple(): : unsigned int offset = bv->bv_offset & (dev->ctrl.page_size - 1); 0.64 : ffff8000107d4ce4: sub w1, w24, #0x1 : iod->dma_len = bv->bv_len; 0.00 : ffff8000107d4ce8: str w26, [x28, #344] : unsigned int offset = bv->bv_offset & (dev->ctrl.page_size - 1); 0.00 : ffff8000107d4cec: and w27, w1, w27 : cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma); 0.00 : ffff8000107d4cf0: str x0, [x29, #208] : unsigned int first_prp_len = dev->ctrl.page_size - offset; 0.00 : ffff8000107d4cf4: sub w24, w24, w27 : if (bv->bv_len > first_prp_len) 0.00 : ffff8000107d4cf8: cmp w24, w26 0.00 : ffff8000107d4cfc: b.cs ffff8000107d4b18 // b.hs, b.nlast : cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len); 0.00 : ffff8000107d4d00: add x24, x0, w24, uxtw 0.00 : ffff8000107d4d04: str x24, [x29, #216] 0.00 : ffff8000107d4d08: b ffff8000107d4b18 : dma_map_page_attrs(): : addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); 0.00 : ffff8000107d4d0c: mov x5, #0x0 // #0 0.00 : ffff8000107d4d10: bl ffff800010163c08 0.00 : ffff8000107d4d14: b ffff8000107d4cd8 : nvme_queue_rq(): : nvme_unmap_data(dev, req); 0.00 : ffff8000107d4d18: mov x1, x28 0.00 : ffff8000107d4d1c: mov x0, x22 : nvme_map_metadata(): : return BLK_STS_IOERR; 0.00 : ffff8000107d4d20: mov w23, #0xa // #10 : nvme_queue_rq(): : nvme_unmap_data(dev, req); 0.00 : ffff8000107d4d24: bl ffff8000107d35e0 0.00 : ffff8000107d4d28: b ffff8000107d4c70 : nvme_setup_prp_simple(): : return BLK_STS_RESOURCE; 0.00 : ffff8000107d4d2c: mov w23, #0x9 // #9 0.00 : ffff8000107d4d30: b ffff8000107d4c70 : nvme_map_data(): : iod->use_sgl = nvme_pci_use_sgls(dev, req); 0.00 : ffff8000107d4d34: strb wzr, [x24, #40] 0.00 : ffff8000107d4d38: ubfiz x4, x1, #5, #16 : blk_rq_payload_bytes(): : if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 0.00 : ffff8000107d4d3c: cbz w7, ffff8000107d5074 : nvme_pci_setup_prps(): : u32 page_size = dev->ctrl.page_size; 0.00 : ffff8000107d4d40: ldr w7, [x22, #1720] : u64 dma_addr = sg_dma_address(sg); 0.00 : ffff8000107d4d44: ldr x5, [x26, #16] : int offset = dma_addr & (page_size - 1); 0.00 : ffff8000107d4d48: sub w27, w7, #0x1 : int dma_len = sg_dma_len(sg); 0.00 : ffff8000107d4d4c: ldr w2, [x26, #24] : blk_rq_payload_bytes(): : return rq->special_vec.bv_len; 0.00 : ffff8000107d4d50: ldr w0, [x28, #112] : nvme_pci_setup_prps(): : int offset = dma_addr & (page_size - 1); 0.00 : ffff8000107d4d54: and w1, w27, w5 0.00 : ffff8000107d4d58: b ffff8000107d4ad4 : nvme_pci_sgl_set_seg(): : sge->type = NVME_SGL_FMT_SEG_DESC << 4; 0.00 : ffff8000107d4d5c: mov x5, x0 : sge->length = cpu_to_le32(PAGE_SIZE); 0.00 : ffff8000107d4d60: mov w0, #0x1000 // #4096 : nvme_pci_setup_sgls(): : sg_list[i++] = *link; 0.00 : ffff8000107d4d64: mov w1, w25 : nvme_pci_sgl_set_seg(): : sge->length = cpu_to_le32(PAGE_SIZE); 0.00 : ffff8000107d4d68: str w0, [x2, #8] : sge->type = NVME_SGL_FMT_SEG_DESC << 4; 0.00 : ffff8000107d4d6c: mov w0, #0x20 // #32 0.00 : ffff8000107d4d70: strb w0, [x2, #15] 0.00 : ffff8000107d4d74: b ffff8000107d49a8 : dma_map_page_attrs(): 0.00 : ffff8000107d4d78: mov x5, #0x0 // #0 0.00 : ffff8000107d4d7c: bl ffff800010163c08 0.00 : ffff8000107d4d80: b ffff8000107d4c38 : rq_integrity_vec(): : if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1)) 0.00 : ffff8000107d4d84: brk #0x800 0.00 : ffff8000107d4d88: ldr x3, [x28] : nvme_map_metadata(): : iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req), 0.00 : ffff8000107d4d8c: mov x2, #0x0 // #0 0.00 : ffff8000107d4d90: ldr x1, [x2] : rq_integrity_vec(): 0.00 : ffff8000107d4d94: ldrh w3, [x3, #1130] 0.00 : ffff8000107d4d98: cmp w3, #0x1 0.00 : ffff8000107d4d9c: b.ls ffff8000107d5090 // b.plast 0.00 : ffff8000107d4da0: brk #0x800 0.00 : ffff8000107d4da4: ldr x3, [x28] : nvme_map_metadata(): 0.00 : ffff8000107d4da8: ldr w2, [x2, #12] : rq_integrity_vec(): 0.00 : ffff8000107d4dac: ldrh w3, [x3, #1130] 0.00 : ffff8000107d4db0: cmp w3, #0x1 0.00 : ffff8000107d4db4: b.ls ffff8000107d507c // b.plast 0.00 : ffff8000107d4db8: brk #0x800 : return NULL; 0.00 : ffff8000107d4dbc: mov x5, #0x0 // #0 0.00 : ffff8000107d4dc0: ldr w3, [x28, #24] 0.00 : ffff8000107d4dc4: b ffff8000107d4b48 0.00 : ffff8000107d4dc8: mov w2, w1 0.00 : ffff8000107d4dcc: b ffff8000107d48e0 : dma_map_sg_attrs(): : BUG_ON(ents < 0); 0.00 : ffff8000107d4dd0: brk #0x800 : nvme_pci_setup_prps(): : iod->first_dma = 0; 0.00 : ffff8000107d4dd4: mov x5, #0x0 // #0 0.00 : ffff8000107d4dd8: str xzr, [x24, #56] 0.00 : ffff8000107d4ddc: b ffff8000107d4b0c : nvme_pci_setup_sgls(): : pool = dev->prp_page_pool; 0.00 : ffff8000107d4de0: ldr x0, [x22, #344] 0.00 : ffff8000107d4de4: str x0, [x29, #136] : iod->npages = 1; 0.00 : ffff8000107d4de8: mov w0, #0x1 // #1 0.00 : ffff8000107d4dec: str w0, [x24, #48] 0.00 : ffff8000107d4df0: b ffff8000107d4948 : nvme_pci_setup_prps(): : nprps = DIV_ROUND_UP(length, page_size); 0.00 : ffff8000107d4df4: add w0, w27, w25 : pool = dev->prp_small_pool; 0.00 : ffff8000107d4df8: ldp x1, x2, [x22, #344] 0.00 : ffff8000107d4dfc: str x2, [x29, #136] : nprps = DIV_ROUND_UP(length, page_size); 0.00 : ffff8000107d4e00: udiv w0, w0, w7 : if (nprps <= (256 / 8)) { 0.00 : ffff8000107d4e04: cmp w0, #0x20 0.00 : ffff8000107d4e08: b.gt ffff8000107d4fa8 : iod->npages = 0; 0.00 : ffff8000107d4e0c: str wzr, [x24, #48] : prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); 0.00 : ffff8000107d4e10: ldr x0, [x29, #136] 0.00 : ffff8000107d4e14: mov w1, #0xa20 // #2592 0.00 : ffff8000107d4e18: str w7, [x29, #104] 0.00 : ffff8000107d4e1c: add x2, x29, #0xa8 0.00 : ffff8000107d4e20: stp x9, x5, [x29, #112] 0.00 : ffff8000107d4e24: str w8, [x29, #128] 0.00 : ffff8000107d4e28: str w3, [x29, #148] 0.00 : ffff8000107d4e2c: str x4, [x29, #152] 0.00 : ffff8000107d4e30: bl ffff800010239bc8 : if (!prp_list) { 0.00 : ffff8000107d4e34: ldr w7, [x29, #104] : prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); 0.00 : ffff8000107d4e38: mov x27, x0 : if (!prp_list) { 0.00 : ffff8000107d4e3c: ldr w8, [x29, #128] 0.00 : ffff8000107d4e40: ldr w3, [x29, #148] 0.00 : ffff8000107d4e44: ldp x9, x5, [x29, #112] 0.00 : ffff8000107d4e48: ldr x4, [x29, #152] 0.00 : ffff8000107d4e4c: cbz x0, ffff8000107d504c : list[0] = prp_list; 0.00 : ffff8000107d4e50: str x0, [x26, x4] : nvme_pci_iod_list(): : return (void **)(iod->sg + blk_rq_nr_phys_segments(req)); 0.00 : ffff8000107d4e54: add x0, x26, x4 : nvme_pci_setup_prps(): : if (i == page_size >> 3) { 0.00 : ffff8000107d4e58: lsr w26, w7, #3 : prp_list[0] = old_prp_list[i - 1]; 0.00 : ffff8000107d4e5c: mov x1, #0xfffffffffffffff8 // #-8 : nvme_pci_iod_list(): : return (void **)(iod->sg + blk_rq_nr_phys_segments(req)); 0.00 : ffff8000107d4e60: str x0, [x29, #104] : nvme_pci_setup_prps(): : i = 0; 0.00 : ffff8000107d4e64: mov w25, #0x0 // #0 : iod->first_dma = prp_dma; 0.00 : ffff8000107d4e68: ldr x0, [x29, #168] 0.00 : ffff8000107d4e6c: str x0, [x24, #56] : prp_list[0] = old_prp_list[i - 1]; 0.00 : ffff8000107d4e70: add x0, x1, w26, sxtw #3 0.00 : ffff8000107d4e74: str x0, [x29, #96] : if (i == page_size >> 3) { 0.00 : ffff8000107d4e78: cmp w26, w25 0.00 : ffff8000107d4e7c: b.eq ffff8000107d4ecc // b.none 0.00 : ffff8000107d4e80: sbfiz x2, x25, #3, #32 0.00 : ffff8000107d4e84: add w25, w25, #0x1 : prp_list[i++] = cpu_to_le64(dma_addr); 0.00 : ffff8000107d4e88: str x5, [x27, x2] 0.00 : ffff8000107d4e8c: sub w8, w8, w7 : dma_len -= page_size; 0.00 : ffff8000107d4e90: sub w3, w3, w7 : if (length <= 0) 0.00 : ffff8000107d4e94: cmp w8, #0x0 0.00 : ffff8000107d4e98: b.le ffff8000107d4fc4 : if (dma_len > 0) 0.00 : ffff8000107d4e9c: cmp w3, #0x0 0.00 : ffff8000107d4ea0: b.gt ffff8000107d4f30 0.00 : ffff8000107d4ea4: stp w7, w8, [x29, #148] : if (unlikely(dma_len < 0)) 0.00 : ffff8000107d4ea8: b.ne ffff8000107d4fb8 // b.any : sg = sg_next(sg); 0.00 : ffff8000107d4eac: mov x0, x9 0.00 : ffff8000107d4eb0: bl ffff8000104867f8 0.00 : ffff8000107d4eb4: ldp w7, w8, [x29, #148] 0.00 : ffff8000107d4eb8: mov x9, x0 : dma_len = sg_dma_len(sg); 0.00 : ffff8000107d4ebc: ldr w3, [x0, #24] : if (i == page_size >> 3) { 0.00 : ffff8000107d4ec0: cmp w26, w25 : dma_addr = sg_dma_address(sg); 0.00 : ffff8000107d4ec4: ldr x5, [x0, #16] : if (i == page_size >> 3) { 0.00 : ffff8000107d4ec8: b.ne ffff8000107d4e80 // b.any : prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); 0.00 : ffff8000107d4ecc: ldr x0, [x29, #136] 0.00 : ffff8000107d4ed0: add x2, x29, #0xa8 0.00 : ffff8000107d4ed4: mov w1, #0xa20 // #2592 0.00 : ffff8000107d4ed8: str w7, [x29, #112] 0.00 : ffff8000107d4edc: stp x9, x5, [x29, #120] 0.00 : ffff8000107d4ee0: stp w8, w3, [x29, #148] 0.00 : ffff8000107d4ee4: bl ffff800010239bc8 : if (!prp_list) 0.00 : ffff8000107d4ee8: cbz x0, ffff8000107d4f94 : list[iod->npages++] = prp_list; 0.00 : ffff8000107d4eec: ldr w4, [x24, #48] : old_prp_list[i - 1] = cpu_to_le64(prp_dma); 0.00 : ffff8000107d4ef0: mov x2, #0x8 // #8 : list[iod->npages++] = prp_list; 0.00 : ffff8000107d4ef4: ldr x1, [x29, #104] : old_prp_list[i - 1] = cpu_to_le64(prp_dma); 0.00 : ffff8000107d4ef8: mov w25, #0x2 // #2 : list[iod->npages++] = prp_list; 0.00 : ffff8000107d4efc: add w11, w4, #0x1 0.00 : ffff8000107d4f00: str w11, [x24, #48] : old_prp_list[i - 1] = cpu_to_le64(prp_dma); 0.00 : ffff8000107d4f04: ldr w7, [x29, #112] : list[iod->npages++] = prp_list; 0.00 : ffff8000107d4f08: str x0, [x1, w4, sxtw #3] : prp_list[0] = old_prp_list[i - 1]; 0.00 : ffff8000107d4f0c: ldr x1, [x29, #96] : old_prp_list[i - 1] = cpu_to_le64(prp_dma); 0.00 : ffff8000107d4f10: ldp w8, w3, [x29, #148] : prp_list[0] = old_prp_list[i - 1]; 0.00 : ffff8000107d4f14: ldr x4, [x27, x1] 0.00 : ffff8000107d4f18: str x4, [x0] : old_prp_list[i - 1] = cpu_to_le64(prp_dma); 0.00 : ffff8000107d4f1c: ldp x9, x5, [x29, #120] 0.00 : ffff8000107d4f20: ldr x4, [x29, #168] 0.00 : ffff8000107d4f24: str x4, [x27, x1] 0.00 : ffff8000107d4f28: mov x27, x0 0.00 : ffff8000107d4f2c: b ffff8000107d4e88 : dma_addr += page_size; 0.00 : ffff8000107d4f30: add x5, x5, w7, uxtw 0.00 : ffff8000107d4f34: b ffff8000107d4e78 : nvme_pci_setup_sgls(): : nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg); 0.00 : ffff8000107d4f38: ldr x0, [x26, #16] : nvme_pci_sgl_set_data(): : sge->addr = cpu_to_le64(sg_dma_address(sg)); 0.00 : ffff8000107d4f3c: str x0, [x29, #208] : sge->length = cpu_to_le32(sg_dma_len(sg)); 0.00 : ffff8000107d4f40: ldr w0, [x26, #24] 0.00 : ffff8000107d4f44: str w0, [x29, #216] : sge->type = NVME_SGL_FMT_DATA_DESC << 4; 0.00 : ffff8000107d4f48: strb wzr, [x29, #223] 0.00 : ffff8000107d4f4c: b ffff8000107d4b18 : nvme_pci_sgl_set_seg(): : sge->length = cpu_to_le32(PAGE_SIZE); 0.00 : ffff8000107d4f50: mov w1, #0x1000 // #4096 : sge->type = NVME_SGL_FMT_SEG_DESC << 4; 0.00 : ffff8000107d4f54: mov w0, #0x20 // #32 : sge->length = cpu_to_le32(PAGE_SIZE); 0.00 : ffff8000107d4f58: str w1, [x29, #216] : sge->type = NVME_SGL_FMT_SEG_DESC << 4; 0.00 : ffff8000107d4f5c: strb w0, [x29, #223] 0.00 : ffff8000107d4f60: b ffff8000107d49a0 0.00 : ffff8000107d4f64: str w7, [x29, #136] : nvme_pci_setup_prps(): : sg = sg_next(sg); 0.00 : ffff8000107d4f68: mov x0, x26 0.00 : ffff8000107d4f6c: str w25, [x29, #148] 0.00 : ffff8000107d4f70: str x4, [x29, #152] 0.00 : ffff8000107d4f74: bl ffff8000104867f8 : dma_len = sg_dma_len(sg); 0.00 : ffff8000107d4f78: ldr w7, [x29, #136] : sg = sg_next(sg); 0.00 : ffff8000107d4f7c: mov x9, x0 : dma_len = sg_dma_len(sg); 0.00 : ffff8000107d4f80: ldr w3, [x0, #24] 0.00 : ffff8000107d4f84: ldr w8, [x29, #148] : dma_addr = sg_dma_address(sg); 0.00 : ffff8000107d4f88: ldr x5, [x0, #16] : dma_len = sg_dma_len(sg); 0.00 : ffff8000107d4f8c: ldr x4, [x29, #152] 0.00 : ffff8000107d4f90: b ffff8000107d4afc : return BLK_STS_RESOURCE; 0.00 : ffff8000107d4f94: mov w23, #0x9 // #9 : nvme_map_data(): : nvme_unmap_data(dev, req); 0.00 : ffff8000107d4f98: mov x1, x28 0.00 : ffff8000107d4f9c: mov x0, x22 0.00 : ffff8000107d4fa0: bl ffff8000107d35e0 0.00 : ffff8000107d4fa4: b ffff8000107d4c70 : nvme_pci_setup_prps(): : iod->npages = 1; 0.00 : ffff8000107d4fa8: mov w0, #0x1 // #1 : pool = dev->prp_page_pool; 0.00 : ffff8000107d4fac: str x1, [x29, #136] : iod->npages = 1; 0.00 : ffff8000107d4fb0: str w0, [x24, #48] 0.00 : ffff8000107d4fb4: b ffff8000107d4e10 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000107d4fb8: b ffff8000107d4fe0 : nvme_pci_setup_prps(): 0.00 : ffff8000107d4fbc: mov w23, #0xa // #10 0.00 : ffff8000107d4fc0: b ffff8000107d4f98 0.00 : ffff8000107d4fc4: ldr x5, [x24, #56] 0.00 : ffff8000107d4fc8: ldr x26, [x24, #80] 0.00 : ffff8000107d4fcc: b ffff8000107d4b0c : nvme_pci_setup_sgls(): : iod->npages = -1; 0.00 : ffff8000107d4fd0: mov w0, #0xffffffff // #-1 0.00 : ffff8000107d4fd4: mov w23, #0x9 // #9 0.00 : ffff8000107d4fd8: str w0, [x24, #48] 0.00 : ffff8000107d4fdc: b ffff8000107d4f98 : nvme_pci_setup_prps(): : WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents), 0.00 : ffff8000107d4fe0: adrp x20, ffff800011b56000 0.00 : ffff8000107d4fe4: add x20, x20, #0x7c8 0.00 : ffff8000107d4fe8: add x20, x20, #0x10 0.00 : ffff8000107d4fec: add x1, x29, #0xb0 0.00 : ffff8000107d4ff0: mov x0, x20 0.00 : ffff8000107d4ff4: bl ffff8000104914b8 <__do_once_start> 0.00 : ffff8000107d4ff8: tst w0, #0xff 0.00 : ffff8000107d4ffc: b.eq ffff8000107d4fbc // b.none 0.00 : ffff8000107d5000: ldr w1, [x24, #52] 0.00 : ffff8000107d5004: ldr x0, [x24, #80] 0.00 : ffff8000107d5008: bl ffff8000107d2fb8 0.00 : ffff8000107d500c: adrp x1, ffff800011a93000 0.00 : ffff8000107d5010: add x1, x1, #0x720 0.00 : ffff8000107d5014: mov x0, x20 0.00 : ffff8000107d5018: add x2, x29, #0xb0 0.00 : ffff8000107d501c: add x1, x1, #0x110 0.00 : ffff8000107d5020: bl ffff800010491520 <__do_once_done> : blk_rq_payload_bytes(): : if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 0.00 : ffff8000107d5024: ldr w0, [x28, #28] 0.00 : ffff8000107d5028: tbz w0, #18, ffff8000107d506c : return rq->special_vec.bv_len; 0.00 : ffff8000107d502c: ldr w1, [x28, #112] : nvme_pci_setup_prps(): 0.00 : ffff8000107d5030: ldr w2, [x24, #52] 0.00 : ffff8000107d5034: adrp x0, ffff800011280000 0.00 : ffff8000107d5038: add x0, x0, #0x8b0 0.00 : ffff8000107d503c: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff8000107d5040: brk #0x800 0.00 : ffff8000107d5044: mov w23, #0xa // #10 0.00 : ffff8000107d5048: b ffff8000107d4f98 : iod->npages = -1; 0.00 : ffff8000107d504c: mov w0, #0xffffffff // #-1 0.00 : ffff8000107d5050: mov w23, #0x9 // #9 0.00 : ffff8000107d5054: str w0, [x24, #48] : iod->first_dma = dma_addr; 0.00 : ffff8000107d5058: str x5, [x24, #56] 0.00 : ffff8000107d505c: b ffff8000107d4f98 0.00 : ffff8000107d5060: stp x24, x25, [x29, #56] 0.00 : ffff8000107d5064: stp x26, x27, [x29, #72] : nvme_queue_rq(): : } 0.00 : ffff8000107d5068: bl ffff8000100e64d0 <__stack_chk_fail> : blk_rq_payload_bytes(): : return blk_rq_bytes(rq); 0.00 : ffff8000107d506c: ldr w1, [x28, #40] 0.00 : ffff8000107d5070: b ffff8000107d5030 0.00 : ffff8000107d5074: ldr w0, [x28, #40] 0.00 : ffff8000107d5078: b ffff8000107d4abc 0.00 : ffff8000107d507c: ldr x4, [x28, #56] 0.00 : ffff8000107d5080: ldr w3, [x28, #24] 0.00 : ffff8000107d5084: ldr x4, [x4, #88] 0.00 : ffff8000107d5088: ldr x5, [x4, #96] 0.00 : ffff8000107d508c: b ffff8000107d4b48 0.00 : ffff8000107d5090: ldr x2, [x28, #56] 0.00 : ffff8000107d5094: ldr w3, [x28, #24] 0.00 : ffff8000107d5098: ldr x2, [x2, #88] 0.00 : ffff8000107d509c: ldr x5, [x2, #96] 0.00 : ffff8000107d50a0: b ffff8000107d4b44 Percent | Source code & Disassembly of vmlinux for cycles (237 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102db238 : : io_submit_one(): : } : } : : static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, : bool compat) : { 1.24 : ffff8000102db238: stp x29, x30, [sp, #-208]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.43 : ffff8000102db23c: mrs x3, sp_el0 : io_submit_one(): 0.00 : ffff8000102db240: mov x29, sp 0.43 : ffff8000102db244: stp x20, x21, [sp, #24] 0.00 : ffff8000102db248: adrp x21, ffff800011909000 1.26 : ffff8000102db24c: stp x22, x23, [sp, #40] 0.00 : ffff8000102db250: add x5, x21, #0x908 0.00 : ffff8000102db254: mov x22, x0 0.00 : ffff8000102db258: mov x20, x1 : __range_ok(): : * Asynchronous I/O running in a kernel thread does not have the : * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag : * the user address before checking. : */ : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && : (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) 0.00 : ffff8000102db25c: ldr w4, [x3, #44] : io_submit_one(): 0.00 : ffff8000102db260: ldr x0, [x5] 0.81 : ffff8000102db264: str x0, [x29, #200] 0.00 : ffff8000102db268: mov x0, #0x0 // #0 0.00 : ffff8000102db26c: and w23, w2, #0xff : __range_ok(): : unsigned long ret, limit = current_thread_info()->addr_limit; 1.71 : ffff8000102db270: ldr x1, [x3, #8] : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 0.00 : ffff8000102db274: tbnz w4, #21, ffff8000102db288 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000102db278: ldr x2, [x3] : __range_ok(): 0.00 : ffff8000102db27c: mov x0, x20 0.00 : ffff8000102db280: tst w2, #0x4000000 0.00 : ffff8000102db284: b.eq ffff8000102db290 // b.none : sign_extend64(): : * @index: 0 based bit index (0<=index<64) to sign bit : */ : static inline __s64 sign_extend64(__u64 value, int index) : { : __u8 shift = 63 - index; : return (__s64)(value << shift) >> shift; 0.00 : ffff8000102db288: sbfx x0, x20, #0, #56 : __range_ok(): : addr = untagged_addr(addr); 0.00 : ffff8000102db28c: and x0, x20, x0 : : __chk_user_ptr(addr); : asm volatile( 0.00 : ffff8000102db290: adds x0, x0, #0x40 0.00 : ffff8000102db294: csel x1, xzr, x1, hi // hi = pmore 0.00 : ffff8000102db298: csinv x0, x0, xzr, cc // cc = lo, ul, last 0.00 : ffff8000102db29c: sbcs xzr, x0, x1 0.00 : ffff8000102db2a0: cset x0, ls // ls = plast : _copy_from_user(): : static inline __must_check unsigned long : _copy_from_user(void *to, const void __user *from, unsigned long n) : { : unsigned long res = n; : might_fault(); : if (likely(access_ok(from, n))) { 0.00 : ffff8000102db2a4: cbz x0, ffff8000102db980 1.27 : ffff8000102db2a8: str x25, [x29, #64] : uaccess_enable_not_uao(): : __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); : } : : static inline void uaccess_enable_not_uao(void) : { : __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102db2ac: nop : sign_extend64(): 7.99 : ffff8000102db2b0: sbfx x0, x20, #0, #56 : get_current(): 0.00 : ffff8000102db2b4: mrs x25, sp_el0 : __uaccess_mask_ptr(): : asm volatile( : " bics xzr, %3, %2\n" : " csel %0, %1, xzr, eq\n" : : "=&r" (safe_ptr) : : "r" (ptr), "r" (current_thread_info()->addr_limit), : "r" (untagged_addr(ptr)) 0.00 : ffff8000102db2b8: and x0, x20, x0 : asm volatile( 0.00 : ffff8000102db2bc: ldr x2, [x25, #8] 0.00 : ffff8000102db2c0: bics xzr, x0, x2 0.00 : ffff8000102db2c4: csel x1, x20, xzr, eq // eq = none : : "cc"); : : csdb(); 0.00 : ffff8000102db2c8: csdb : _copy_from_user(): : kasan_check_write(to, n); : res = raw_copy_from_user(to, from, n); 0.00 : ffff8000102db2cc: mov x2, #0x40 // #64 0.00 : ffff8000102db2d0: add x0, x29, #0x88 0.00 : ffff8000102db2d4: bl ffff800010cb7140 <__arch_copy_from_user> : uaccess_disable_not_uao(): : __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102db2d8: nop : _copy_from_user(): : } : if (unlikely(res)) 0.00 : ffff8000102db2dc: cbnz x0, ffff8000102db978 : io_submit_one(): : : if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb)))) : return -EFAULT; : : /* enforce forwards compatibility on users */ : if (unlikely(iocb.aio_reserved2)) { 5.88 : ffff8000102db2e0: ldr x0, [x29, #184] 0.00 : ffff8000102db2e4: cbnz x0, ffff8000102db96c : pr_debug("EINVAL: reserve field set\n"); : return -EINVAL; : } : : /* prevent overflows */ : if (unlikely( 8.00 : ffff8000102db2e8: ldr x0, [x29, #168] 0.00 : ffff8000102db2ec: tbnz x0, #63, ffff8000102db96c 4.63 : ffff8000102db2f0: str x24, [x29, #56] : aio_get_req(): : req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL); 0.00 : ffff8000102db2f4: adrp x24, ffff800011b33000 0.00 : ffff8000102db2f8: add x0, x24, #0x1a0 2.98 : ffff8000102db2fc: str x19, [x29, #16] 0.00 : ffff8000102db300: mov w1, #0xcc0 // #3264 0.00 : ffff8000102db304: ldr x0, [x0, #8] 0.00 : ffff8000102db308: bl ffff800010252238 0.43 : ffff8000102db30c: mov x19, x0 : if (unlikely(!req)) 0.00 : ffff8000102db310: cbz x0, ffff8000102db988 : get_reqs_available(): : if (__get_reqs_available(ctx)) 0.00 : ffff8000102db314: mov x0, x22 0.00 : ffff8000102db318: str x26, [x29, #72] 0.00 : ffff8000102db31c: bl ffff8000102d8fa8 <__get_reqs_available> 0.00 : ffff8000102db320: tst w0, #0xff 0.00 : ffff8000102db324: b.eq ffff8000102db47c // b.none : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000102db328: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000102db32c: ldr x0, [x22, #72] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000102db330: tst x0, #0x3 0.00 : ffff8000102db334: b.ne ffff8000102dbac0 // b.any : get_current(): 0.00 : ffff8000102db338: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000102db33c: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000102db340: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db344: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102db348: mov x3, #0x1 // #1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000102db34c: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102db350: add x0, x0, x2 0.00 : ffff8000102db354: ldxr x5, [x0] 1.62 : ffff8000102db358: add x5, x5, x3 0.00 : ffff8000102db35c: stxr w4, x5, [x0] 0.00 : ffff8000102db360: cbnz w4, ffff8000102db354 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db364: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102db368: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db36c: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102db370: cbnz x0, ffff8000102db470 : percpu_ref_get_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_add(*percpu_count, nr); 0.00 : ffff8000102db374: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff8000102db378: bl ffff800010160d20 <__rcu_read_unlock> : __io_submit_one(): : req->ki_filp = fget(iocb->aio_fildes); 2.13 : ffff8000102db37c: mov x25, x19 0.00 : ffff8000102db380: ldr w0, [x29, #156] : __write_once_size(): 0.00 : ffff8000102db384: mov w1, #0x2 // #2 : aio_get_req(): : INIT_LIST_HEAD(&req->ki_list); 0.00 : ffff8000102db388: add x26, x19, #0x90 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000102db38c: str x26, [x19, #144] : aio_get_req(): : req->ki_ctx = ctx; 0.00 : ffff8000102db390: str x22, [x19, #96] : INIT_LIST_HEAD(): : * the result is an empty list. : */ : static inline void INIT_LIST_HEAD(struct list_head *list) : { : WRITE_ONCE(list->next, list); : list->prev = list; 0.00 : ffff8000102db394: str x26, [x19, #152] : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db398: str w1, [x19, #160] : aio_get_req(): : req->ki_eventfd = NULL; 0.43 : ffff8000102db39c: str xzr, [x19, #168] : __io_submit_one(): : req->ki_filp = fget(iocb->aio_fildes); 0.00 : ffff8000102db3a0: bl ffff8000102a1510 1.28 : ffff8000102db3a4: str x0, [x25], #160 : if (unlikely(!req->ki_filp)) 0.00 : ffff8000102db3a8: cbz x0, ffff8000102db998 : if (iocb->aio_flags & IOCB_FLAG_RESFD) { 0.00 : ffff8000102db3ac: ldr w0, [x29, #192] 0.00 : ffff8000102db3b0: tbz w0, #0, ffff8000102db3c8 : eventfd = eventfd_ctx_fdget(iocb->aio_resfd); 0.00 : ffff8000102db3b4: ldr w0, [x29, #196] 0.00 : ffff8000102db3b8: bl ffff8000102d7eb0 : if (IS_ERR(eventfd)) 0.00 : ffff8000102db3bc: cmn x0, #0x1, lsl #12 0.00 : ffff8000102db3c0: b.hi ffff8000102db774 // b.pmore : req->ki_eventfd = eventfd; 0.00 : ffff8000102db3c4: str x0, [x19, #168] : get_current(): 0.00 : ffff8000102db3c8: mrs x0, sp_el0 : __range_ok(): : (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) 0.00 : ffff8000102db3cc: ldr w3, [x0, #44] : __io_submit_one(): : if (unlikely(put_user(KIOCB_KEY, &user_iocb->aio_key))) { 0.00 : ffff8000102db3d0: add x2, x20, #0x8 : __range_ok(): : unsigned long ret, limit = current_thread_info()->addr_limit; 1.68 : ffff8000102db3d4: ldr x1, [x0, #8] : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 0.00 : ffff8000102db3d8: tbnz w3, #21, ffff8000102db3ec : test_bit(): 0.00 : ffff8000102db3dc: ldr x3, [x0] : __range_ok(): 0.00 : ffff8000102db3e0: mov x0, x2 0.00 : ffff8000102db3e4: tst w3, #0x4000000 0.00 : ffff8000102db3e8: b.eq ffff8000102db3f4 // b.none : sign_extend64(): 0.00 : ffff8000102db3ec: sbfx x0, x2, #0, #56 : __range_ok(): : addr = untagged_addr(addr); 0.00 : ffff8000102db3f0: and x0, x2, x0 : asm volatile( 0.00 : ffff8000102db3f4: adds x0, x0, #0x4 0.00 : ffff8000102db3f8: csel x1, xzr, x1, hi // hi = pmore 0.00 : ffff8000102db3fc: csinv x0, x0, xzr, cc // cc = lo, ul, last 0.00 : ffff8000102db400: sbcs xzr, x0, x1 0.00 : ffff8000102db404: cset x0, ls // ls = plast : __io_submit_one(): 0.00 : ffff8000102db408: cbnz x0, ffff8000102db708 : return -EFAULT; 0.00 : ffff8000102db40c: mov w20, #0xfffffff2 // #-14 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.43 : ffff8000102db410: b ffff8000102db4cc 0.00 : ffff8000102db414: b ffff8000102db4cc : __lse_atomic_fetch_sub_release(): : return i; \ : } : : ATOMIC_FETCH_OP_SUB(_relaxed, ) : ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") : ATOMIC_FETCH_OP_SUB(_release, l, "memory") 0.00 : ffff8000102db418: mov w1, #0x1 // #1 0.00 : ffff8000102db41c: add x0, x19, #0xa0 0.00 : ffff8000102db420: neg w1, w1 0.00 : ffff8000102db424: ldaddl w1, w1, [x0] : refcount_sub_and_test(): : */ : static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) : { : int old = atomic_fetch_sub_release(i, &r->refs); : : if (old == i) { 42.64 : ffff8000102db428: cmp w1, #0x1 0.00 : ffff8000102db42c: b.eq ffff8000102db4e0 // b.none : smp_acquire__after_ctrl_dep(); : return true; : } : : if (unlikely(old < 0 || old - i < 0)) 0.00 : ffff8000102db430: cmp w1, #0x0 0.00 : ffff8000102db434: b.le ffff8000102db808 : io_submit_one(): : /* : * If err is 0, we'd either done aio_complete() ourselves or have : * arranged for that to be done asynchronously. Anything non-zero : * means that we need to destroy req ourselves. : */ : if (unlikely(err)) { 0.85 : ffff8000102db438: cbnz w20, ffff8000102db9a0 0.85 : ffff8000102db43c: ldr x19, [x29, #16] 0.00 : ffff8000102db440: ldp x24, x25, [x29, #56] 0.00 : ffff8000102db444: ldr x26, [x29, #72] : iocb_destroy(req); : put_reqs_available(ctx, 1); : } : return err; : } 0.00 : ffff8000102db448: add x21, x21, #0x908 0.00 : ffff8000102db44c: mov w0, w20 0.00 : ffff8000102db450: ldr x2, [x29, #200] 1.71 : ffff8000102db454: ldr x1, [x21] 0.00 : ffff8000102db458: eor x1, x2, x1 0.00 : ffff8000102db45c: cbnz x1, ffff8000102dbea8 0.00 : ffff8000102db460: ldp x20, x21, [sp, #24] 0.00 : ffff8000102db464: ldp x22, x23, [sp, #40] 0.00 : ffff8000102db468: ldp x29, x30, [sp], #208 0.00 : ffff8000102db46c: ret : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db470: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102db474: cbz x0, ffff8000102db374 0.42 : ffff8000102db478: b ffff8000102db378 : spin_lock_irq(): : raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \ : } while (0) : : static __always_inline void spin_lock_irq(spinlock_t *lock) : { : raw_spin_lock_irq(&lock->rlock); 0.00 : ffff8000102db47c: add x26, x22, #0x1c8 0.00 : ffff8000102db480: mov x0, x26 0.00 : ffff8000102db484: bl ffff800010cd8460 <_raw_spin_lock_irq> : user_refill_reqs_available(): : if (ctx->completed_events) { 0.00 : ffff8000102db488: ldr w0, [x22, #452] 0.00 : ffff8000102db48c: cbnz w0, ffff8000102db788 : spin_unlock_irq(): : raw_spin_unlock_bh(&lock->rlock); : } : : static __always_inline void spin_unlock_irq(spinlock_t *lock) : { : raw_spin_unlock_irq(&lock->rlock); 0.00 : ffff8000102db490: mov x0, x26 0.00 : ffff8000102db494: bl ffff800010cd7c70 <_raw_spin_unlock_irq> : get_reqs_available(): : return __get_reqs_available(ctx); 0.00 : ffff8000102db498: mov x0, x22 0.00 : ffff8000102db49c: bl ffff8000102d8fa8 <__get_reqs_available> : aio_get_req(): : if (unlikely(!get_reqs_available(ctx))) { 0.00 : ffff8000102db4a0: tst w0, #0xff 0.00 : ffff8000102db4a4: b.ne ffff8000102db328 // b.any : kmem_cache_free(kiocb_cachep, req); 0.00 : ffff8000102db4a8: add x24, x24, #0x1a0 0.00 : ffff8000102db4ac: mov x1, x19 : io_submit_one(): : return -EAGAIN; 0.00 : ffff8000102db4b0: mov w20, #0xfffffff5 // #-11 : aio_get_req(): : kmem_cache_free(kiocb_cachep, req); 0.00 : ffff8000102db4b4: ldr x0, [x24, #8] 0.00 : ffff8000102db4b8: bl ffff800010253518 0.00 : ffff8000102db4bc: ldr x19, [x29, #16] 0.00 : ffff8000102db4c0: ldp x24, x25, [x29, #56] 0.00 : ffff8000102db4c4: ldr x26, [x29, #72] 0.00 : ffff8000102db4c8: b ffff8000102db448 : __ll_sc_atomic_fetch_sub_release(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000102db4cc: mov w0, #0x1 // #1 0.00 : ffff8000102db4d0: add x4, x19, #0xa0 0.00 : ffff8000102db4d4: b ffff8000102dda4c <__arm64_compat_sys_io_pgetevents_time64+0x3dc> : refcount_sub_and_test(): : if (old == i) { 0.00 : ffff8000102db4d8: cmp w1, #0x1 0.00 : ffff8000102db4dc: b.ne ffff8000102db430 // b.any 0.00 : ffff8000102db4e0: stp x27, x28, [x29, #80] : smp_acquire__after_ctrl_dep(); 0.00 : ffff8000102db4e4: dmb ishld : aio_complete(): : struct kioctx *ctx = iocb->ki_ctx; 0.00 : ffff8000102db4e8: ldr x23, [x19, #96] : spinlock_check(): : return &lock->rlock; 0.00 : ffff8000102db4ec: add x26, x23, #0x1c8 : aio_complete(): : spin_lock_irqsave(&ctx->completion_lock, flags); 0.00 : ffff8000102db4f0: mov x0, x26 0.00 : ffff8000102db4f4: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : pos = tail + AIO_EVENTS_OFFSET; 0.85 : ffff8000102db4f8: ldr w2, [x23, #448] : spin_lock_irqsave(&ctx->completion_lock, flags); 0.00 : ffff8000102db4fc: mov x27, x0 : if (++tail >= ctx->nr_events) 0.43 : ffff8000102db500: ldr w4, [x23, #144] : pos = tail + AIO_EVENTS_OFFSET; 0.00 : ffff8000102db504: add w2, w2, #0x1 : ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.00 : ffff8000102db508: ldr x0, [x23, #168] : get_current(): 0.00 : ffff8000102db50c: mrs x1, sp_el0 : aio_complete(): 0.00 : ffff8000102db510: lsr w28, w2, #7 : __read_once_size(): 0.00 : ffff8000102db514: ldr w3, [x1, #16] : aio_complete(): : if (++tail >= ctx->nr_events) 0.00 : ffff8000102db518: cmp w2, w4 : __preempt_count_add(): : pc += val; 0.00 : ffff8000102db51c: add w3, w3, #0x1 : aio_complete(): 0.00 : ffff8000102db520: csel w25, w2, wzr, cc // cc = lo, ul, last : ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.00 : ffff8000102db524: ldr x0, [x0, x28, lsl #3] : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db528: str w3, [x1, #16] : pagefault_disabled_inc(): : } : #endif : : static __always_inline void pagefault_disabled_inc(void) : { : current->pagefault_disabled++; 0.00 : ffff8000102db52c: ldr w3, [x1, #2448] 0.00 : ffff8000102db530: add w3, w3, #0x1 0.00 : ffff8000102db534: str w3, [x1, #2448] : lowmem_page_address(): : */ : #include : : static __always_inline void *lowmem_page_address(const struct page *page) : { : return page_to_virt(page); 0.00 : ffff8000102db538: mov x3, #0x200000 // #2097152 : aio_complete(): : event = ev_page + pos % AIO_EVENTS_PER_PAGE; 0.00 : ffff8000102db53c: ubfiz x4, x2, #5, #7 : lowmem_page_address(): 0.00 : ffff8000102db540: movk x3, #0x200, lsl #32 0.00 : ffff8000102db544: add x0, x0, x3 : aio_complete(): : *event = iocb->ki_res; 0.00 : ffff8000102db548: mov x5, #0xffff000000000000 // #-281474976710656 : lowmem_page_address(): 0.00 : ffff8000102db54c: lsr x0, x0, #6 : aio_complete(): 0.00 : ffff8000102db550: ldp x2, x3, [x19, #112] 0.00 : ffff8000102db554: add x0, x4, x0, lsl #12 0.00 : ffff8000102db558: add x0, x0, x5 0.00 : ffff8000102db55c: stp x2, x3, [x0] 0.42 : ffff8000102db560: ldp x2, x3, [x19, #128] 0.00 : ffff8000102db564: stp x2, x3, [x0, #16] : pagefault_disabled_dec(): : } : : static __always_inline void pagefault_disabled_dec(void) : { : current->pagefault_disabled--; 0.00 : ffff8000102db568: ldr w0, [x1, #2448] 0.00 : ffff8000102db56c: sub w0, w0, #0x1 0.00 : ffff8000102db570: str w0, [x1, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db574: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102db578: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db57c: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102db580: cbnz x0, ffff8000102db77c : __kunmap_atomic(): : #define kmap_atomic_prot(page, prot) kmap_atomic(page) : : static inline void __kunmap_atomic(void *addr) : { : pagefault_enable(); : preempt_enable(); 0.00 : ffff8000102db584: bl ffff800010cd2e50 : aio_complete(): : flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.00 : ffff8000102db588: ldr x0, [x23, #168] 0.00 : ffff8000102db58c: ldr x0, [x0, x28, lsl #3] 0.00 : ffff8000102db590: bl ffff8000100a3110 : smp_wmb(); /* make event visible before updating tail */ 0.00 : ffff8000102db594: dmb ishst : ctx->tail = tail; 0.85 : ffff8000102db598: str w25, [x23, #448] : ring = kmap_atomic(ctx->ring_pages[0]); 0.00 : ffff8000102db59c: ldr x1, [x23, #168] : get_current(): 0.00 : ffff8000102db5a0: mrs x0, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db5a4: ldr w2, [x0, #16] : aio_complete(): 0.00 : ffff8000102db5a8: ldr x1, [x1] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102db5ac: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db5b0: str w2, [x0, #16] : pagefault_disabled_inc(): : current->pagefault_disabled++; 0.00 : ffff8000102db5b4: ldr w2, [x0, #2448] 0.00 : ffff8000102db5b8: add w2, w2, #0x1 0.00 : ffff8000102db5bc: str w2, [x0, #2448] : lowmem_page_address(): 0.00 : ffff8000102db5c0: mov x2, #0x200000 // #2097152 0.00 : ffff8000102db5c4: movk x2, #0x200, lsl #32 0.00 : ffff8000102db5c8: add x1, x1, x2 0.00 : ffff8000102db5cc: mov x2, #0xffff000000000000 // #-281474976710656 0.00 : ffff8000102db5d0: lsr x1, x1, #6 0.00 : ffff8000102db5d4: add x1, x2, x1, lsl #12 : aio_complete(): : head = ring->head; 0.00 : ffff8000102db5d8: ldr w28, [x1, #8] : ring->tail = tail; 0.00 : ffff8000102db5dc: str w25, [x1, #12] : pagefault_disabled_dec(): : current->pagefault_disabled--; 0.00 : ffff8000102db5e0: ldr w1, [x0, #2448] 0.00 : ffff8000102db5e4: sub w1, w1, #0x1 0.42 : ffff8000102db5e8: str w1, [x0, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db5ec: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102db5f0: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db5f4: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102db5f8: cbz x1, ffff8000102db800 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db5fc: ldr x0, [x0, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102db600: cbz x0, ffff8000102db800 : aio_complete(): : flush_dcache_page(ctx->ring_pages[0]); 0.00 : ffff8000102db604: ldr x0, [x23, #168] 0.00 : ffff8000102db608: ldr x0, [x0] 0.00 : ffff8000102db60c: bl ffff8000100a3110 : ctx->completed_events++; 0.00 : ffff8000102db610: ldr w0, [x23, #452] 0.00 : ffff8000102db614: add w0, w0, #0x1 0.42 : ffff8000102db618: str w0, [x23, #452] : if (ctx->completed_events > 1) 0.00 : ffff8000102db61c: cmp w0, #0x1 0.00 : ffff8000102db620: b.ls ffff8000102db634 // b.plast : refill_reqs_available(ctx, head, tail); 0.00 : ffff8000102db624: mov w2, w25 0.00 : ffff8000102db628: mov w1, w28 0.00 : ffff8000102db62c: mov x0, x23 0.00 : ffff8000102db630: bl ffff8000102d8f58 : spin_unlock_irqrestore(): : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000102db634: mov x0, x26 0.00 : ffff8000102db638: mov x1, x27 0.00 : ffff8000102db63c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : aio_complete(): : if (iocb->ki_eventfd) 0.00 : ffff8000102db640: ldr x0, [x19, #168] 0.00 : ffff8000102db644: cbz x0, ffff8000102db650 : eventfd_signal(iocb->ki_eventfd, 1); 0.00 : ffff8000102db648: mov x1, #0x1 // #1 0.00 : ffff8000102db64c: bl ffff8000102d81c0 : smp_mb(); 0.00 : ffff8000102db650: dmb ish : __read_once_size(): 0.43 : ffff8000102db654: ldr x1, [x23, #424] : waitqueue_active(): : * Also note that this 'optimization' trades a spin_lock() for an smp_mb(), : * which (when the lock is uncontended) are of roughly equal cost. : */ : static inline int waitqueue_active(struct wait_queue_head *wq_head) : { : return !list_empty(&wq_head->head); 0.00 : ffff8000102db658: add x2, x23, #0x1a8 0.00 : ffff8000102db65c: add x0, x23, #0x1a0 : aio_complete(): : if (waitqueue_active(&ctx->wait)) 0.41 : ffff8000102db660: cmp x2, x1 0.00 : ffff8000102db664: b.eq ffff8000102db678 // b.none : wake_up(&ctx->wait); 0.00 : ffff8000102db668: mov x3, #0x0 // #0 0.00 : ffff8000102db66c: mov w2, #0x1 // #1 0.00 : ffff8000102db670: mov w1, #0x3 // #3 0.00 : ffff8000102db674: bl ffff80001012f9c0 <__wake_up> : iocb_destroy(): : if (iocb->ki_eventfd) 0.00 : ffff8000102db678: ldr x0, [x19, #168] 0.00 : ffff8000102db67c: cbz x0, ffff8000102db684 : eventfd_ctx_put(iocb->ki_eventfd); 0.00 : ffff8000102db680: bl ffff8000102d7d50 : if (iocb->ki_filp) 0.00 : ffff8000102db684: ldr x0, [x19] 0.00 : ffff8000102db688: cbz x0, ffff8000102db690 : fput(iocb->ki_filp); 0.00 : ffff8000102db68c: bl ffff80001027df98 : percpu_ref_put(&iocb->ki_ctx->reqs); 0.00 : ffff8000102db690: ldr x23, [x19, #96] : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff8000102db694: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.00 : ffff8000102db698: ldr x0, [x23, #72] : __ref_is_percpu(): : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000102db69c: tst x0, #0x3 0.00 : ffff8000102db6a0: b.ne ffff8000102dbb70 // b.any : get_current(): 0.00 : ffff8000102db6a4: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000102db6a8: ldr w2, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102db6ac: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db6b0: str w2, [x1, #16] : __percpu_add_case_64(): 0.00 : ffff8000102db6b4: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000102db6b8: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102db6bc: add x0, x0, x2 0.00 : ffff8000102db6c0: ldxr x5, [x0] 0.00 : ffff8000102db6c4: add x5, x5, x3 0.00 : ffff8000102db6c8: stxr w4, x5, [x0] 0.00 : ffff8000102db6cc: cbnz w4, ffff8000102db6c0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db6d0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102db6d4: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db6d8: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102db6dc: cbz x0, ffff8000102db6e8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db6e0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102db6e4: cbnz x0, ffff8000102db6ec : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff8000102db6e8: bl ffff800010cd2e78 : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff8000102db6ec: bl ffff800010160d20 <__rcu_read_unlock> : iocb_destroy(): : kmem_cache_free(kiocb_cachep, iocb); 0.00 : ffff8000102db6f0: add x0, x24, #0x1a0 0.00 : ffff8000102db6f4: mov x1, x19 0.00 : ffff8000102db6f8: ldr x0, [x0, #8] 0.00 : ffff8000102db6fc: bl ffff800010253518 0.00 : ffff8000102db700: ldp x27, x28, [x29, #80] 0.00 : ffff8000102db704: b ffff8000102db438 : sign_extend64(): 0.43 : ffff8000102db708: sbfx x0, x2, #0, #56 : get_current(): 0.42 : ffff8000102db70c: mrs x1, sp_el0 : __uaccess_mask_ptr(): : "r" (untagged_addr(ptr)) 0.00 : ffff8000102db710: and x0, x2, x0 : asm volatile( 0.00 : ffff8000102db714: ldr x3, [x1, #8] 0.00 : ffff8000102db718: bics xzr, x0, x3 0.00 : ffff8000102db71c: csel x1, x2, xzr, eq // eq = none : csdb(); 0.00 : ffff8000102db720: csdb : uaccess_enable_not_uao(): : __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); 0.42 : ffff8000102db724: nop : __io_submit_one(): : if (unlikely(put_user(KIOCB_KEY, &user_iocb->aio_key))) { 0.00 : ffff8000102db728: mov w0, #0x0 // #0 0.43 : ffff8000102db72c: str w0, [x1] : uaccess_disable_not_uao(): : __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102db730: nop : __io_submit_one(): 0.00 : ffff8000102db734: cbnz w0, ffff8000102db40c : req->ki_res.data = iocb->aio_data; 0.85 : ffff8000102db738: ldr x1, [x29, #136] : switch (iocb->aio_lio_opcode) { 0.00 : ffff8000102db73c: ldrh w0, [x29, #152] : req->ki_res.obj = (u64)(unsigned long)user_iocb; 0.00 : ffff8000102db740: stp x1, x20, [x19, #112] : req->ki_res.res2 = 0; 0.42 : ffff8000102db744: stp xzr, xzr, [x19, #128] : switch (iocb->aio_lio_opcode) { 0.00 : ffff8000102db748: cmp w0, #0x3 0.00 : ffff8000102db74c: b.eq ffff8000102dbaa8 // b.none 0.42 : ffff8000102db750: b.hi ffff8000102db818 // b.pmore 0.00 : ffff8000102db754: cmp w0, #0x1 0.00 : ffff8000102db758: b.eq ffff8000102dba8c // b.none 0.00 : ffff8000102db75c: b.hi ffff8000102dba74 // b.pmore : return aio_read(&req->rw, iocb, false, compat); 0.00 : ffff8000102db760: mov w3, w23 0.00 : ffff8000102db764: mov w2, #0x0 // #0 0.00 : ffff8000102db768: add x1, x29, #0x88 0.00 : ffff8000102db76c: mov x0, x19 0.00 : ffff8000102db770: bl ffff8000102d8930 1.28 : ffff8000102db774: mov w20, w0 0.00 : ffff8000102db778: b ffff8000102db410 : __read_once_size(): 0.00 : ffff8000102db77c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102db780: cbz x0, ffff8000102db584 0.00 : ffff8000102db784: b ffff8000102db588 0.00 : ffff8000102db788: str x27, [x29, #80] : user_refill_reqs_available(): : ring = kmap_atomic(ctx->ring_pages[0]); 0.00 : ffff8000102db78c: ldr x0, [x22, #168] : __read_once_size(): 0.00 : ffff8000102db790: ldr w1, [x25, #16] : user_refill_reqs_available(): 0.00 : ffff8000102db794: ldr x0, [x0] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102db798: add w1, w1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db79c: str w1, [x25, #16] : pagefault_disabled_inc(): : current->pagefault_disabled++; 0.00 : ffff8000102db7a0: ldr w1, [x25, #2448] 0.00 : ffff8000102db7a4: add w1, w1, #0x1 0.00 : ffff8000102db7a8: str w1, [x25, #2448] : lowmem_page_address(): 0.00 : ffff8000102db7ac: mov x1, #0x200000 // #2097152 0.00 : ffff8000102db7b0: movk x1, #0x200, lsl #32 0.00 : ffff8000102db7b4: add x0, x0, x1 : user_refill_reqs_available(): : head = ring->head; 0.00 : ffff8000102db7b8: mov x1, #0xffff000000000000 // #-281474976710656 : lowmem_page_address(): 0.00 : ffff8000102db7bc: lsr x0, x0, #6 : user_refill_reqs_available(): 0.00 : ffff8000102db7c0: add x0, x1, x0, lsl #12 0.00 : ffff8000102db7c4: ldr w27, [x0, #8] : pagefault_disabled_dec(): : current->pagefault_disabled--; 0.00 : ffff8000102db7c8: ldr w0, [x25, #2448] 0.00 : ffff8000102db7cc: sub w0, w0, #0x1 0.00 : ffff8000102db7d0: str w0, [x25, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db7d4: ldr x0, [x25, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102db7d8: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db7dc: str w0, [x25, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102db7e0: cbnz x0, ffff8000102dbae4 : __kunmap_atomic(): 0.00 : ffff8000102db7e4: bl ffff800010cd2e50 : user_refill_reqs_available(): : refill_reqs_available(ctx, head, ctx->tail); 0.00 : ffff8000102db7e8: ldr w2, [x22, #448] 0.00 : ffff8000102db7ec: mov w1, w27 0.00 : ffff8000102db7f0: mov x0, x22 0.00 : ffff8000102db7f4: bl ffff8000102d8f58 0.00 : ffff8000102db7f8: ldr x27, [x29, #80] 0.00 : ffff8000102db7fc: b ffff8000102db490 : __kunmap_atomic(): 0.00 : ffff8000102db800: bl ffff800010cd2e50 0.00 : ffff8000102db804: b ffff8000102db604 : refcount_sub_and_test(): : refcount_warn_saturate(r, REFCOUNT_SUB_UAF); 0.00 : ffff8000102db808: mov w1, #0x3 // #3 0.00 : ffff8000102db80c: mov x0, x25 0.00 : ffff8000102db810: bl ffff8000104915e0 0.00 : ffff8000102db814: b ffff8000102db438 : __io_submit_one(): : switch (iocb->aio_lio_opcode) { 0.00 : ffff8000102db818: cmp w0, #0x7 0.00 : ffff8000102db81c: b.eq ffff8000102dba58 // b.none 0.00 : ffff8000102db820: cmp w0, #0x8 0.00 : ffff8000102db824: b.eq ffff8000102dba3c // b.none : return -EINVAL; 0.00 : ffff8000102db828: mov w20, #0xffffffea // #-22 : switch (iocb->aio_lio_opcode) { 0.00 : ffff8000102db82c: cmp w0, #0x5 0.00 : ffff8000102db830: b.ne ffff8000102db410 // b.any : aio_poll(): : if ((u16)iocb->aio_buf != iocb->aio_buf) 0.00 : ffff8000102db834: ldr x2, [x29, #160] 0.00 : ffff8000102db838: stp x27, x28, [x29, #80] : struct kioctx *ctx = aiocb->ki_ctx; 0.00 : ffff8000102db83c: ldr x20, [x19, #96] : if ((u16)iocb->aio_buf != iocb->aio_buf) 0.00 : ffff8000102db840: tst x2, #0xffffffffffff0000 0.00 : ffff8000102db844: b.ne ffff8000102dbe2c // b.any : if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags) 0.00 : ffff8000102db848: ldr x0, [x29, #176] 0.00 : ffff8000102db84c: cbnz x0, ffff8000102dbe2c 0.00 : ffff8000102db850: ldr x0, [x29, #168] 0.00 : ffff8000102db854: cbnz x0, ffff8000102dbe2c 0.00 : ffff8000102db858: ldr w0, [x29, #148] 0.00 : ffff8000102db85c: cbnz w0, ffff8000102dbe2c : INIT_WORK(&req->work, aio_poll_complete_work); 0.00 : ffff8000102db860: add x3, x19, #0x48 : req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; 0.00 : ffff8000102db864: mov w1, #0x27ff // #10239 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000102db868: str x3, [x19, #72] : aio_poll(): 0.00 : ffff8000102db86c: and w1, w1, w2 : mask = vfs_poll(req->file, &apt.pt) & req->events; 0.00 : ffff8000102db870: ldr x0, [x19] : INIT_WORK(&req->work, aio_poll_complete_work); 0.00 : ffff8000102db874: adrp x2, ffff8000102da000 0.00 : ffff8000102db878: add x2, x2, #0x750 : req->head = NULL; 0.00 : ffff8000102db87c: str xzr, [x19, #8] : req->done = false; 0.00 : ffff8000102db880: strb wzr, [x19, #20] : req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; 0.00 : ffff8000102db884: orr w1, w1, #0x18 : INIT_LIST_HEAD(&req->wait.entry); 0.00 : ffff8000102db888: add x23, x19, #0x30 : req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; 0.00 : ffff8000102db88c: str w1, [x19, #16] : req->cancelled = false; 0.00 : ffff8000102db890: strb wzr, [x19, #21] : init_waitqueue_func_entry(): : wq_entry->flags = 0; 0.00 : ffff8000102db894: str wzr, [x19, #24] : wq_entry->private = NULL; 0.00 : ffff8000102db898: str xzr, [x19, #32] : __write_once_size(): 0.00 : ffff8000102db89c: str x23, [x19, #48] : INIT_LIST_HEAD(): 0.00 : ffff8000102db8a0: str x23, [x19, #56] : aio_poll(): : INIT_WORK(&req->work, aio_poll_complete_work); 0.00 : ffff8000102db8a4: stp x3, x2, [x19, #80] 0.00 : ffff8000102db8a8: mov x3, #0xfffffffe0 // #68719476704 : init_waitqueue_func_entry(): : wq_entry->func = func; 0.00 : ffff8000102db8ac: adrp x2, ffff8000102db000 : aio_poll(): 0.00 : ffff8000102db8b0: str x3, [x19, #64] : init_waitqueue_func_entry(): 0.00 : ffff8000102db8b4: add x2, x2, #0xec0 0.00 : ffff8000102db8b8: str x2, [x19, #40] : aio_poll(): : apt.pt._qproc = aio_poll_queue_proc; 0.00 : ffff8000102db8bc: adrp x2, ffff8000102d8000 0.00 : ffff8000102db8c0: add x2, x2, #0x678 0.00 : ffff8000102db8c4: str x2, [x29, #104] : apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */ 0.00 : ffff8000102db8c8: mov w3, #0xffffffea // #-22 : vfs_poll(): : return file->f_op->poll; : } : : static inline __poll_t vfs_poll(struct file *file, struct poll_table_struct *pt) : { : if (unlikely(!file->f_op->poll)) 0.00 : ffff8000102db8cc: ldr x2, [x0, #40] : aio_poll(): : apt.pt._key = req->events; 0.00 : ffff8000102db8d0: str w1, [x29, #112] : apt.iocb = aiocb; 0.00 : ffff8000102db8d4: str x19, [x29, #120] : vfs_poll(): 0.00 : ffff8000102db8d8: ldr x2, [x2, #72] : aio_poll(): : apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */ 0.00 : ffff8000102db8dc: str w3, [x29, #128] : vfs_poll(): 0.00 : ffff8000102db8e0: cbz x2, ffff8000102dbe38 : return DEFAULT_POLLMASK; : return file->f_op->poll(file, pt); 0.00 : ffff8000102db8e4: add x1, x29, #0x68 0.00 : ffff8000102db8e8: blr x2 0.00 : ffff8000102db8ec: ldr w1, [x19, #16] : spin_lock_irq(): : raw_spin_lock_irq(&lock->rlock); 0.00 : ffff8000102db8f0: add x27, x20, #0x140 : aio_poll(): : mask = vfs_poll(req->file, &apt.pt) & req->events; 0.00 : ffff8000102db8f4: and w28, w0, w1 : spin_lock_irq(): 0.00 : ffff8000102db8f8: mov x0, x27 0.00 : ffff8000102db8fc: bl ffff800010cd8460 <_raw_spin_lock_irq> : aio_poll(): : if (likely(req->head)) { 0.00 : ffff8000102db900: ldr x0, [x19, #8] 0.00 : ffff8000102db904: cbz x0, ffff8000102dbb18 : spin_lock(): : raw_spin_lock(&lock->rlock); 0.00 : ffff8000102db908: bl ffff800010cd80f0 <_raw_spin_lock> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db90c: ldr x0, [x19, #48] : aio_poll(): : if (unlikely(list_empty(&req->wait.entry))) { 0.00 : ffff8000102db910: cmp x23, x0 0.00 : ffff8000102db914: b.eq ffff8000102dbe40 // b.none : if (mask || apt.error) { 0.00 : ffff8000102db918: cbnz w28, ffff8000102dbafc 0.00 : ffff8000102db91c: ldr w0, [x29, #128] 0.00 : ffff8000102db920: cbnz w0, ffff8000102dbafc : } else if (!req->done) { /* actually waiting for an event */ 0.00 : ffff8000102db924: ldrb w0, [x19, #20] 0.00 : ffff8000102db928: cbnz w0, ffff8000102dbe60 : list_add_tail(): : * Insert a new entry before the specified head. : * This is useful for implementing queues. : */ : static inline void list_add_tail(struct list_head *new, struct list_head *head) : { : __list_add(new, head->prev, head); 0.00 : ffff8000102db92c: add x2, x20, #0x200 0.00 : ffff8000102db930: ldr x1, [x20, #336] : aio_poll(): : list_add_tail(&aiocb->ki_list, &ctx->active_reqs); 0.00 : ffff8000102db934: add x20, x20, #0x148 : aiocb->ki_cancel = aio_poll_cancel; 0.00 : ffff8000102db938: adrp x0, ffff8000102d8000 0.00 : ffff8000102db93c: add x0, x0, #0x608 : __list_add(): : next->prev = new; 0.00 : ffff8000102db940: stur x26, [x2, #-176] : new->prev = prev; 0.00 : ffff8000102db944: stp x20, x1, [x19, #144] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000102db948: str x26, [x1] : aio_poll(): 0.00 : ffff8000102db94c: str x0, [x19, #104] : spin_unlock(): : raw_spin_unlock(&lock->rlock); 0.00 : ffff8000102db950: ldr x0, [x19, #8] 0.00 : ffff8000102db954: bl ffff800010cd7c38 <_raw_spin_unlock> : spin_unlock_irq(): : raw_spin_unlock_irq(&lock->rlock); 0.00 : ffff8000102db958: mov x0, x27 0.00 : ffff8000102db95c: bl ffff800010cd7c70 <_raw_spin_unlock_irq> 0.00 : ffff8000102db960: ldp x27, x28, [x29, #80] : aio_poll(): : return apt.error; 0.00 : ffff8000102db964: ldr w20, [x29, #128] 0.00 : ffff8000102db968: b ffff8000102db410 : io_submit_one(): : return -EINVAL; 0.00 : ffff8000102db96c: mov w20, #0xffffffea // #-22 0.00 : ffff8000102db970: ldr x25, [x29, #64] 0.00 : ffff8000102db974: b ffff8000102db448 0.00 : ffff8000102db978: ldr x25, [x29, #64] 0.00 : ffff8000102db97c: nop : return -EFAULT; 0.00 : ffff8000102db980: mov w20, #0xfffffff2 // #-14 0.00 : ffff8000102db984: b ffff8000102db448 : return -EAGAIN; 0.00 : ffff8000102db988: mov w20, #0xfffffff5 // #-11 0.00 : ffff8000102db98c: ldr x19, [x29, #16] 0.00 : ffff8000102db990: ldp x24, x25, [x29, #56] 0.00 : ffff8000102db994: b ffff8000102db448 : __io_submit_one(): : return -EBADF; 0.00 : ffff8000102db998: mov w20, #0xfffffff7 // #-9 0.00 : ffff8000102db99c: b ffff8000102db410 : iocb_destroy(): : if (iocb->ki_eventfd) 0.00 : ffff8000102db9a0: ldr x0, [x19, #168] 0.00 : ffff8000102db9a4: cbz x0, ffff8000102db9ac : eventfd_ctx_put(iocb->ki_eventfd); 0.00 : ffff8000102db9a8: bl ffff8000102d7d50 : if (iocb->ki_filp) 0.00 : ffff8000102db9ac: ldr x0, [x19] 0.00 : ffff8000102db9b0: cbz x0, ffff8000102db9b8 : fput(iocb->ki_filp); 0.00 : ffff8000102db9b4: bl ffff80001027df98 : percpu_ref_put(&iocb->ki_ctx->reqs); 0.00 : ffff8000102db9b8: ldr x23, [x19, #96] : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff8000102db9bc: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db9c0: ldr x0, [x23, #72] : __ref_is_percpu(): : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000102db9c4: tst x0, #0x3 0.00 : ffff8000102db9c8: b.ne ffff8000102dbdf0 // b.any : get_current(): 0.00 : ffff8000102db9cc: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000102db9d0: ldr w2, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102db9d4: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102db9d8: str w2, [x1, #16] : __percpu_add_case_64(): 0.00 : ffff8000102db9dc: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000102db9e0: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102db9e4: add x0, x0, x2 0.00 : ffff8000102db9e8: ldxr x5, [x0] 0.00 : ffff8000102db9ec: add x5, x5, x3 0.00 : ffff8000102db9f0: stxr w4, x5, [x0] 0.00 : ffff8000102db9f4: cbnz w4, ffff8000102db9e8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102db9f8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102db9fc: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dba00: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102dba04: cbnz x0, ffff8000102dbaf0 : percpu_ref_put_many(): : this_cpu_sub(*percpu_count, nr); 0.00 : ffff8000102dba08: bl ffff800010cd2e78 : iocb_destroy(): : kmem_cache_free(kiocb_cachep, iocb); 0.00 : ffff8000102dba0c: add x24, x24, #0x1a0 : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff8000102dba10: bl ffff800010160d20 <__rcu_read_unlock> : iocb_destroy(): 0.00 : ffff8000102dba14: mov x1, x19 0.00 : ffff8000102dba18: ldr x0, [x24, #8] 0.00 : ffff8000102dba1c: bl ffff800010253518 : io_submit_one(): : put_reqs_available(ctx, 1); 0.00 : ffff8000102dba20: mov w1, #0x1 // #1 0.00 : ffff8000102dba24: mov x0, x22 0.00 : ffff8000102dba28: bl ffff8000102d8ec8 0.00 : ffff8000102dba2c: ldr x19, [x29, #16] 0.00 : ffff8000102dba30: ldp x24, x25, [x29, #56] 0.00 : ffff8000102dba34: ldr x26, [x29, #72] 0.00 : ffff8000102dba38: b ffff8000102db448 : __io_submit_one(): : return aio_write(&req->rw, iocb, true, compat); 0.00 : ffff8000102dba3c: mov w3, w23 0.00 : ffff8000102dba40: mov w2, #0x1 // #1 0.00 : ffff8000102dba44: add x1, x29, #0x88 0.00 : ffff8000102dba48: mov x0, x19 0.00 : ffff8000102dba4c: bl ffff8000102d8aa0 0.00 : ffff8000102dba50: mov w20, w0 0.00 : ffff8000102dba54: b ffff8000102db410 : return aio_read(&req->rw, iocb, true, compat); 0.00 : ffff8000102dba58: mov w3, w23 0.00 : ffff8000102dba5c: mov w2, #0x1 // #1 0.00 : ffff8000102dba60: add x1, x29, #0x88 0.00 : ffff8000102dba64: mov x0, x19 0.00 : ffff8000102dba68: bl ffff8000102d8930 0.00 : ffff8000102dba6c: mov w20, w0 0.00 : ffff8000102dba70: b ffff8000102db410 : return aio_fsync(&req->fsync, iocb, false); 0.00 : ffff8000102dba74: mov w2, #0x0 // #0 0.00 : ffff8000102dba78: add x1, x29, #0x88 0.00 : ffff8000102dba7c: mov x0, x19 0.00 : ffff8000102dba80: bl ffff8000102d8c38 0.00 : ffff8000102dba84: mov w20, w0 0.00 : ffff8000102dba88: b ffff8000102db410 : return aio_write(&req->rw, iocb, false, compat); 0.00 : ffff8000102dba8c: mov w3, w23 0.00 : ffff8000102dba90: mov w2, #0x0 // #0 0.00 : ffff8000102dba94: add x1, x29, #0x88 0.00 : ffff8000102dba98: mov x0, x19 0.00 : ffff8000102dba9c: bl ffff8000102d8aa0 0.00 : ffff8000102dbaa0: mov w20, w0 0.00 : ffff8000102dbaa4: b ffff8000102db410 : return aio_fsync(&req->fsync, iocb, true); 0.00 : ffff8000102dbaa8: mov w2, #0x1 // #1 0.00 : ffff8000102dbaac: add x1, x29, #0x88 0.00 : ffff8000102dbab0: mov x0, x19 0.00 : ffff8000102dbab4: bl ffff8000102d8c38 0.00 : ffff8000102dbab8: mov w20, w0 0.00 : ffff8000102dbabc: b ffff8000102db410 : percpu_ref_get_many(): : atomic_long_add(nr, &ref->count); 0.00 : ffff8000102dbac0: add x1, x22, #0x40 : arch_static_branch_jump(): 0.00 : ffff8000102dbac4: b ffff8000102dbad8 0.00 : ffff8000102dbac8: b ffff8000102dbad8 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff8000102dbacc: mov x0, #0x1 // #1 0.00 : ffff8000102dbad0: stadd x0, [x1] 0.00 : ffff8000102dbad4: b ffff8000102db378 : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff8000102dbad8: add x2, x22, #0x40 0.00 : ffff8000102dbadc: b ffff8000102dda64 <__arm64_compat_sys_io_pgetevents_time64+0x3f4> 0.00 : ffff8000102dbae0: b ffff8000102db378 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dbae4: ldr x0, [x25, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102dbae8: cbz x0, ffff8000102db7e4 0.00 : ffff8000102dbaec: b ffff8000102db7e8 : __read_once_size(): 0.00 : ffff8000102dbaf0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102dbaf4: cbz x0, ffff8000102dba08 0.00 : ffff8000102dbaf8: b ffff8000102dba0c : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff8000102dbafc: ldp x1, x0, [x19, #48] : __list_del(): : next->prev = prev; 0.00 : ffff8000102dbb00: str x0, [x1, #8] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000102dbb04: str x1, [x0] 0.00 : ffff8000102dbb08: str x23, [x19, #48] : spin_unlock(): : raw_spin_unlock(&lock->rlock); 0.00 : ffff8000102dbb0c: ldr x0, [x19, #8] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff8000102dbb10: str x23, [x19, #56] : spin_unlock(): 0.00 : ffff8000102dbb14: bl ffff800010cd7c38 <_raw_spin_unlock> : aio_poll(): : if (mask) { /* no async, we'd stolen it */ 0.00 : ffff8000102dbb18: cbz w28, ffff8000102db958 : mangle_poll(): : { : __u16 v = (__force __u16)val; : #define M(X) __MAP(v, (__force __u16)EPOLL##X, POLL##X) : return M(IN) | M(OUT) | M(PRI) | M(ERR) | M(NVAL) | : M(RDNORM) | M(RDBAND) | M(WRNORM) | M(WRBAND) | : M(HUP) | M(RDHUP) | M(MSG); 0.00 : ffff8000102dbb1c: mov w0, #0x27ff // #10239 0.00 : ffff8000102dbb20: and w0, w28, w0 : aio_poll(): : aiocb->ki_res.res = mangle_poll(mask); 0.00 : ffff8000102dbb24: and x0, x0, #0xffff : apt.error = 0; 0.00 : ffff8000102dbb28: str wzr, [x29, #128] : aiocb->ki_res.res = mangle_poll(mask); 0.00 : ffff8000102dbb2c: str x0, [x19, #128] : spin_unlock_irq(): : raw_spin_unlock_irq(&lock->rlock); 0.00 : ffff8000102dbb30: mov x0, x27 0.00 : ffff8000102dbb34: bl ffff800010cd7c70 <_raw_spin_unlock_irq> : arch_static_branch_jump(): 0.00 : ffff8000102dbb38: b ffff8000102dbbac 0.00 : ffff8000102dbb3c: b ffff8000102dbbac : __lse_atomic_fetch_sub_release(): : ATOMIC_FETCH_OP_SUB(_release, l, "memory") 0.00 : ffff8000102dbb40: mov w0, #0x1 // #1 0.00 : ffff8000102dbb44: add x1, x19, #0xa0 0.00 : ffff8000102dbb48: neg w0, w0 0.00 : ffff8000102dbb4c: ldaddl w0, w0, [x1] : refcount_sub_and_test(): : if (old == i) { 0.00 : ffff8000102dbb50: cmp w0, #0x1 0.00 : ffff8000102dbb54: b.eq ffff8000102dbbbc // b.none : if (unlikely(old < 0 || old - i < 0)) 0.00 : ffff8000102dbb58: cmp w0, #0x0 0.00 : ffff8000102dbb5c: b.gt ffff8000102db960 : refcount_warn_saturate(r, REFCOUNT_SUB_UAF); 0.00 : ffff8000102dbb60: mov w1, #0x3 // #3 0.00 : ffff8000102dbb64: mov x0, x25 0.00 : ffff8000102dbb68: bl ffff8000104915e0 0.00 : ffff8000102dbb6c: b ffff8000102db960 : iocb_destroy(): : percpu_ref_put(&iocb->ki_ctx->reqs); 0.00 : ffff8000102dbb70: add x0, x23, #0x40 : arch_static_branch_jump(): 0.00 : ffff8000102dbb74: b ffff8000102dbb9c 0.00 : ffff8000102dbb78: b ffff8000102dbb9c : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000102dbb7c: mov x1, #0x1 // #1 0.00 : ffff8000102dbb80: neg x1, x1 0.00 : ffff8000102dbb84: ldaddal x1, x2, [x0] 0.00 : ffff8000102dbb88: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000102dbb8c: cbnz x1, ffff8000102db6ec : ref->release(ref); 0.00 : ffff8000102dbb90: ldr x1, [x0, #16] 0.00 : ffff8000102dbb94: blr x1 0.00 : ffff8000102dbb98: b ffff8000102db6ec : __ll_sc_atomic64_sub_return(): : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff8000102dbb9c: mov x2, #0x1 // #1 0.00 : ffff8000102dbba0: add x4, x23, #0x40 0.00 : ffff8000102dbba4: b ffff8000102dda7c <__arm64_compat_sys_io_pgetevents_time64+0x40c> 0.00 : ffff8000102dbba8: b ffff8000102dbb8c : __ll_sc_atomic_fetch_sub_release(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000102dbbac: mov w1, #0x1 // #1 0.00 : ffff8000102dbbb0: add x4, x19, #0xa0 0.00 : ffff8000102dbbb4: b ffff8000102dda98 <__arm64_compat_sys_io_pgetevents_time64+0x428> 0.00 : ffff8000102dbbb8: b ffff8000102dbb50 : refcount_sub_and_test(): : smp_acquire__after_ctrl_dep(); 0.00 : ffff8000102dbbbc: dmb ishld : aio_complete(): : struct kioctx *ctx = iocb->ki_ctx; 0.00 : ffff8000102dbbc0: ldr x20, [x19, #96] : spinlock_check(): : return &lock->rlock; 0.00 : ffff8000102dbbc4: add x26, x20, #0x1c8 : aio_complete(): : spin_lock_irqsave(&ctx->completion_lock, flags); 0.00 : ffff8000102dbbc8: mov x0, x26 0.00 : ffff8000102dbbcc: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : pos = tail + AIO_EVENTS_OFFSET; 0.00 : ffff8000102dbbd0: ldr w2, [x20, #448] : spin_lock_irqsave(&ctx->completion_lock, flags); 0.00 : ffff8000102dbbd4: mov x27, x0 : if (++tail >= ctx->nr_events) 0.00 : ffff8000102dbbd8: ldr w4, [x20, #144] : pos = tail + AIO_EVENTS_OFFSET; 0.00 : ffff8000102dbbdc: add w2, w2, #0x1 : ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.00 : ffff8000102dbbe0: ldr x0, [x20, #168] : get_current(): 0.00 : ffff8000102dbbe4: mrs x1, sp_el0 : aio_complete(): 0.00 : ffff8000102dbbe8: lsr w28, w2, #7 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dbbec: ldr w3, [x1, #16] : aio_complete(): : if (++tail >= ctx->nr_events) 0.00 : ffff8000102dbbf0: cmp w2, w4 : __preempt_count_add(): : pc += val; 0.00 : ffff8000102dbbf4: add w3, w3, #0x1 : aio_complete(): 0.00 : ffff8000102dbbf8: csel w23, w2, wzr, cc // cc = lo, ul, last : ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.00 : ffff8000102dbbfc: ldr x0, [x0, x28, lsl #3] : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dbc00: str w3, [x1, #16] : pagefault_disabled_inc(): : current->pagefault_disabled++; 0.00 : ffff8000102dbc04: ldr w3, [x1, #2448] 0.00 : ffff8000102dbc08: add w3, w3, #0x1 0.00 : ffff8000102dbc0c: str w3, [x1, #2448] : lowmem_page_address(): 0.00 : ffff8000102dbc10: mov x3, #0x200000 // #2097152 : aio_complete(): : event = ev_page + pos % AIO_EVENTS_PER_PAGE; 0.00 : ffff8000102dbc14: ubfiz x4, x2, #5, #7 : lowmem_page_address(): 0.00 : ffff8000102dbc18: movk x3, #0x200, lsl #32 0.00 : ffff8000102dbc1c: add x0, x0, x3 : aio_complete(): : *event = iocb->ki_res; 0.00 : ffff8000102dbc20: mov x5, #0xffff000000000000 // #-281474976710656 : lowmem_page_address(): 0.00 : ffff8000102dbc24: lsr x0, x0, #6 : aio_complete(): 0.00 : ffff8000102dbc28: ldp x2, x3, [x19, #112] 0.00 : ffff8000102dbc2c: add x0, x4, x0, lsl #12 0.00 : ffff8000102dbc30: add x0, x0, x5 0.00 : ffff8000102dbc34: stp x2, x3, [x0] 0.00 : ffff8000102dbc38: ldp x2, x3, [x19, #128] 0.00 : ffff8000102dbc3c: stp x2, x3, [x0, #16] : pagefault_disabled_dec(): : current->pagefault_disabled--; 0.00 : ffff8000102dbc40: ldr w0, [x1, #2448] 0.00 : ffff8000102dbc44: sub w0, w0, #0x1 0.00 : ffff8000102dbc48: str w0, [x1, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dbc4c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102dbc50: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dbc54: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102dbc58: cbz x0, ffff8000102dbde8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dbc5c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102dbc60: cbz x0, ffff8000102dbde8 : aio_complete(): : flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 0.00 : ffff8000102dbc64: ldr x0, [x20, #168] 0.00 : ffff8000102dbc68: ldr x0, [x0, x28, lsl #3] 0.00 : ffff8000102dbc6c: bl ffff8000100a3110 : smp_wmb(); /* make event visible before updating tail */ 0.00 : ffff8000102dbc70: dmb ishst : ctx->tail = tail; 0.00 : ffff8000102dbc74: str w23, [x20, #448] : ring = kmap_atomic(ctx->ring_pages[0]); 0.00 : ffff8000102dbc78: ldr x1, [x20, #168] : get_current(): 0.00 : ffff8000102dbc7c: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff8000102dbc80: ldr w2, [x0, #16] : aio_complete(): 0.00 : ffff8000102dbc84: ldr x1, [x1] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102dbc88: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dbc8c: str w2, [x0, #16] : pagefault_disabled_inc(): : current->pagefault_disabled++; 0.00 : ffff8000102dbc90: ldr w2, [x0, #2448] 0.00 : ffff8000102dbc94: add w2, w2, #0x1 0.00 : ffff8000102dbc98: str w2, [x0, #2448] : lowmem_page_address(): 0.00 : ffff8000102dbc9c: mov x2, #0x200000 // #2097152 0.00 : ffff8000102dbca0: movk x2, #0x200, lsl #32 0.00 : ffff8000102dbca4: add x1, x1, x2 0.00 : ffff8000102dbca8: mov x2, #0xffff000000000000 // #-281474976710656 0.00 : ffff8000102dbcac: lsr x1, x1, #6 0.00 : ffff8000102dbcb0: add x1, x2, x1, lsl #12 : aio_complete(): : head = ring->head; 0.00 : ffff8000102dbcb4: ldr w28, [x1, #8] : ring->tail = tail; 0.00 : ffff8000102dbcb8: str w23, [x1, #12] : pagefault_disabled_dec(): : current->pagefault_disabled--; 0.00 : ffff8000102dbcbc: ldr w1, [x0, #2448] 0.00 : ffff8000102dbcc0: sub w1, w1, #0x1 0.00 : ffff8000102dbcc4: str w1, [x0, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dbcc8: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102dbccc: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dbcd0: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102dbcd4: cbz x1, ffff8000102dbde0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dbcd8: ldr x0, [x0, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102dbcdc: cbz x0, ffff8000102dbde0 : aio_complete(): : flush_dcache_page(ctx->ring_pages[0]); 0.00 : ffff8000102dbce0: ldr x0, [x20, #168] 0.00 : ffff8000102dbce4: ldr x0, [x0] 0.00 : ffff8000102dbce8: bl ffff8000100a3110 : ctx->completed_events++; 0.00 : ffff8000102dbcec: ldr w0, [x20, #452] 0.00 : ffff8000102dbcf0: add w0, w0, #0x1 0.00 : ffff8000102dbcf4: str w0, [x20, #452] : if (ctx->completed_events > 1) 0.00 : ffff8000102dbcf8: cmp w0, #0x1 0.00 : ffff8000102dbcfc: b.ls ffff8000102dbd10 // b.plast : refill_reqs_available(ctx, head, tail); 0.00 : ffff8000102dbd00: mov w2, w23 0.00 : ffff8000102dbd04: mov w1, w28 0.00 : ffff8000102dbd08: mov x0, x20 0.00 : ffff8000102dbd0c: bl ffff8000102d8f58 : spin_unlock_irqrestore(): : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000102dbd10: mov x0, x26 0.00 : ffff8000102dbd14: mov x1, x27 0.00 : ffff8000102dbd18: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : aio_complete(): : if (iocb->ki_eventfd) 0.00 : ffff8000102dbd1c: ldr x0, [x19, #168] 0.00 : ffff8000102dbd20: cbz x0, ffff8000102dbd2c : eventfd_signal(iocb->ki_eventfd, 1); 0.00 : ffff8000102dbd24: mov x1, #0x1 // #1 0.00 : ffff8000102dbd28: bl ffff8000102d81c0 : smp_mb(); 0.00 : ffff8000102dbd2c: dmb ish : __read_once_size(): 0.00 : ffff8000102dbd30: ldr x1, [x20, #424] : waitqueue_active(): : return !list_empty(&wq_head->head); 0.00 : ffff8000102dbd34: add x2, x20, #0x1a8 0.00 : ffff8000102dbd38: add x0, x20, #0x1a0 : aio_complete(): : if (waitqueue_active(&ctx->wait)) 0.00 : ffff8000102dbd3c: cmp x2, x1 0.00 : ffff8000102dbd40: b.eq ffff8000102dbd54 // b.none : wake_up(&ctx->wait); 0.00 : ffff8000102dbd44: mov x3, #0x0 // #0 0.00 : ffff8000102dbd48: mov w2, #0x1 // #1 0.00 : ffff8000102dbd4c: mov w1, #0x3 // #3 0.00 : ffff8000102dbd50: bl ffff80001012f9c0 <__wake_up> : iocb_destroy(): : if (iocb->ki_eventfd) 0.00 : ffff8000102dbd54: ldr x0, [x19, #168] 0.00 : ffff8000102dbd58: cbz x0, ffff8000102dbd60 : eventfd_ctx_put(iocb->ki_eventfd); 0.00 : ffff8000102dbd5c: bl ffff8000102d7d50 : if (iocb->ki_filp) 0.00 : ffff8000102dbd60: ldr x0, [x19] 0.00 : ffff8000102dbd64: cbz x0, ffff8000102dbd6c : fput(iocb->ki_filp); 0.00 : ffff8000102dbd68: bl ffff80001027df98 : percpu_ref_put(&iocb->ki_ctx->reqs); 0.00 : ffff8000102dbd6c: ldr x20, [x19, #96] : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff8000102dbd70: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.00 : ffff8000102dbd74: ldr x0, [x20, #72] : __ref_is_percpu(): : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000102dbd78: tst x0, #0x3 0.00 : ffff8000102dbd7c: b.ne ffff8000102dbe6c // b.any : get_current(): 0.00 : ffff8000102dbd80: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000102dbd84: ldr w2, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000102dbd88: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dbd8c: str w2, [x1, #16] : __percpu_add_case_64(): 0.00 : ffff8000102dbd90: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000102dbd94: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102dbd98: add x0, x0, x2 0.00 : ffff8000102dbd9c: ldxr x5, [x0] 0.00 : ffff8000102dbda0: add x5, x5, x3 0.00 : ffff8000102dbda4: stxr w4, x5, [x0] 0.00 : ffff8000102dbda8: cbnz w4, ffff8000102dbd9c : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dbdac: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102dbdb0: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dbdb4: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102dbdb8: cbz x0, ffff8000102dbdc4 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dbdbc: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102dbdc0: cbnz x0, ffff8000102dbdc8 : percpu_ref_put_many(): : this_cpu_sub(*percpu_count, nr); 0.00 : ffff8000102dbdc4: bl ffff800010cd2e78 : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff8000102dbdc8: bl ffff800010160d20 <__rcu_read_unlock> : iocb_destroy(): : kmem_cache_free(kiocb_cachep, iocb); 0.00 : ffff8000102dbdcc: add x0, x24, #0x1a0 0.00 : ffff8000102dbdd0: mov x1, x19 0.00 : ffff8000102dbdd4: ldr x0, [x0, #8] 0.00 : ffff8000102dbdd8: bl ffff800010253518 0.00 : ffff8000102dbddc: b ffff8000102db960 : __kunmap_atomic(): 0.00 : ffff8000102dbde0: bl ffff800010cd2e50 0.00 : ffff8000102dbde4: b ffff8000102dbce0 0.00 : ffff8000102dbde8: bl ffff800010cd2e50 0.00 : ffff8000102dbdec: b ffff8000102dbc64 : iocb_destroy(): : percpu_ref_put(&iocb->ki_ctx->reqs); 0.00 : ffff8000102dbdf0: add x0, x23, #0x40 : arch_static_branch_jump(): 0.00 : ffff8000102dbdf4: b ffff8000102dbe1c 0.00 : ffff8000102dbdf8: b ffff8000102dbe1c : __lse_atomic64_sub_return(): 0.00 : ffff8000102dbdfc: mov x1, #0x1 // #1 0.00 : ffff8000102dbe00: neg x1, x1 0.00 : ffff8000102dbe04: ldaddal x1, x2, [x0] 0.00 : ffff8000102dbe08: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000102dbe0c: cbnz x1, ffff8000102dba0c : ref->release(ref); 0.00 : ffff8000102dbe10: ldr x1, [x0, #16] 0.00 : ffff8000102dbe14: blr x1 0.00 : ffff8000102dbe18: b ffff8000102dba0c : __ll_sc_atomic64_sub_return(): : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff8000102dbe1c: mov x2, #0x1 // #1 0.00 : ffff8000102dbe20: add x4, x23, #0x40 0.00 : ffff8000102dbe24: b ffff8000102ddab0 <__arm64_compat_sys_io_pgetevents_time64+0x440> 0.00 : ffff8000102dbe28: b ffff8000102dbe0c : aio_poll(): : return -EINVAL; 0.00 : ffff8000102dbe2c: mov w20, #0xffffffea // #-22 0.00 : ffff8000102dbe30: ldp x27, x28, [x29, #80] 0.00 : ffff8000102dbe34: b ffff8000102db410 : vfs_poll(): : return DEFAULT_POLLMASK; 0.00 : ffff8000102dbe38: mov w0, #0x145 // #325 0.00 : ffff8000102dbe3c: b ffff8000102db8f0 : aio_poll(): : if (apt.error) 0.00 : ffff8000102dbe40: ldr w0, [x29, #128] 0.00 : ffff8000102dbe44: cbz w0, ffff8000102db924 : spin_unlock(): : raw_spin_unlock(&lock->rlock); 0.00 : ffff8000102dbe48: ldr x0, [x19, #8] : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff8000102dbe4c: mov w1, #0x1 // #1 : aio_poll(): : apt.error = 0; 0.00 : ffff8000102dbe50: str wzr, [x29, #128] : __write_once_size(): 0.00 : ffff8000102dbe54: strb w1, [x19, #21] : spin_unlock(): 0.00 : ffff8000102dbe58: bl ffff800010cd7c38 <_raw_spin_unlock> 0.00 : ffff8000102dbe5c: b ffff8000102db958 0.00 : ffff8000102dbe60: ldr x0, [x19, #8] 0.00 : ffff8000102dbe64: bl ffff800010cd7c38 <_raw_spin_unlock> 0.00 : ffff8000102dbe68: b ffff8000102db958 : iocb_destroy(): : percpu_ref_put(&iocb->ki_ctx->reqs); 0.00 : ffff8000102dbe6c: add x0, x20, #0x40 : arch_static_branch_jump(): 0.00 : ffff8000102dbe70: b ffff8000102dbe98 0.00 : ffff8000102dbe74: b ffff8000102dbe98 : __lse_atomic64_sub_return(): 0.00 : ffff8000102dbe78: mov x1, #0x1 // #1 0.00 : ffff8000102dbe7c: neg x1, x1 0.00 : ffff8000102dbe80: ldaddal x1, x2, [x0] 0.00 : ffff8000102dbe84: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000102dbe88: cbnz x1, ffff8000102dbdc8 : ref->release(ref); 0.00 : ffff8000102dbe8c: ldr x1, [x0, #16] 0.00 : ffff8000102dbe90: blr x1 0.00 : ffff8000102dbe94: b ffff8000102dbdc8 : __ll_sc_atomic64_sub_return(): 0.00 : ffff8000102dbe98: mov x2, #0x1 // #1 0.00 : ffff8000102dbe9c: add x4, x20, #0x40 0.00 : ffff8000102dbea0: b ffff8000102ddacc <__arm64_compat_sys_io_pgetevents_time64+0x45c> 0.00 : ffff8000102dbea4: b ffff8000102dbe88 0.00 : ffff8000102dbea8: str x19, [x29, #16] 0.00 : ffff8000102dbeac: stp x24, x25, [x29, #56] 0.00 : ffff8000102dbeb0: stp x26, x27, [x29, #72] 0.00 : ffff8000102dbeb4: str x28, [x29, #88] : io_submit_one(): : } 0.00 : ffff8000102dbeb8: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (174 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001045ad10 : : generic_make_request_checks(): : return ret; : } : : static noinline_for_stack bool : generic_make_request_checks(struct bio *bio) : { 1.16 : ffff80001045ad10: stp x29, x30, [sp, #-112]! 0.00 : ffff80001045ad14: mov x29, sp 4.05 : ffff80001045ad18: stp x19, x20, [sp, #16] 0.00 : ffff80001045ad1c: adrp x20, ffff800011909000 0.00 : ffff80001045ad20: stp x21, x22, [sp, #32] 0.00 : ffff80001045ad24: add x1, x20, #0x908 0.00 : ffff80001045ad28: mov x19, x0 : blk_status_t status = BLK_STS_IOERR; : char b[BDEVNAME_SIZE]; : : might_sleep(); : : q = bio->bi_disk->queue; 2.33 : ffff80001045ad2c: ldr x22, [x0, #8] : { 0.00 : ffff80001045ad30: ldr x2, [x1] 1.77 : ffff80001045ad34: str x2, [x29, #104] 0.00 : ffff80001045ad38: mov x2, #0x0 // #0 : q = bio->bi_disk->queue; 0.00 : ffff80001045ad3c: ldr x21, [x22, #1040] : if (unlikely(!q)) { 0.58 : ffff80001045ad40: cbz x21, ffff80001045b09c : /* : * Non-mq queues do not honor REQ_NOWAIT, so complete a bio : * with BLK_STS_AGAIN status in order to catch -EAGAIN and : * to give a chance to the caller to repeat request gracefully. : */ : if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_mq(q)) { 0.00 : ffff80001045ad44: ldr w1, [x0, #16] 0.00 : ffff80001045ad48: tbz w1, #21, ffff80001045ad54 0.00 : ffff80001045ad4c: ldr x0, [x21, #48] 0.00 : ffff80001045ad50: cbz x0, ffff80001045b058 : status = BLK_STS_AGAIN; : goto end_io; : } : : if (should_fail_bio(bio)) 0.00 : ffff80001045ad54: bl ffff80001045a7c8 0.58 : ffff80001045ad58: cbz w0, ffff80001045ad94 : blk_status_t status = BLK_STS_IOERR; 0.00 : ffff80001045ad5c: mov w0, #0xa // #10 : return true; : : not_supported: : status = BLK_STS_NOTSUPP; : end_io: : bio->bi_status = status; 0.00 : ffff80001045ad60: strb w0, [x19, #26] : bio_endio(bio); 0.00 : ffff80001045ad64: mov x0, x19 0.00 : ffff80001045ad68: bl ffff800010456b18 : return false; 0.00 : ffff80001045ad6c: mov w0, #0x0 // #0 : } 0.00 : ffff80001045ad70: add x20, x20, #0x908 1.16 : ffff80001045ad74: ldr x2, [x29, #104] 0.00 : ffff80001045ad78: ldr x1, [x20] 0.00 : ffff80001045ad7c: eor x1, x2, x1 0.00 : ffff80001045ad80: cbnz x1, ffff80001045b1f0 0.00 : ffff80001045ad84: ldp x19, x20, [sp, #16] 0.57 : ffff80001045ad88: ldp x21, x22, [sp, #32] 1.76 : ffff80001045ad8c: ldp x29, x30, [sp], #112 0.00 : ffff80001045ad90: ret 0.00 : ffff80001045ad94: str x23, [x29, #48] : if (bio->bi_partno) { 2.33 : ffff80001045ad98: ldrb w0, [x19, #27] : int nr_sectors = bio_sectors(bio); 0.00 : ffff80001045ad9c: ldr w23, [x19, #40] 0.00 : ffff80001045ada0: lsr w23, w23, #9 : if (bio->bi_partno) { 0.00 : ffff80001045ada4: cbz w0, ffff80001045af94 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001045ada8: bl ffff80001015c1b0 <__rcu_read_lock> : blk_partition_remap(): : p = __disk_get_part(bio->bi_disk, bio->bi_partno); 0.00 : ffff80001045adac: ldrb w1, [x19, #27] 0.00 : ffff80001045adb0: ldr x0, [x19, #8] 0.00 : ffff80001045adb4: bl ffff800010470f40 <__disk_get_part> 0.00 : ffff80001045adb8: mov x22, x0 : if (unlikely(!p)) 0.00 : ffff80001045adbc: cbz x0, ffff80001045b174 : bio_check_ro(): : if (part->policy && op_is_write(op)) { 0.00 : ffff80001045adc0: ldr w0, [x0, #816] 0.00 : ffff80001045adc4: cbz w0, ffff80001045ae18 : const int op = bio_op(bio); 0.00 : ffff80001045adc8: ldr w0, [x19, #16] : if (part->policy && op_is_write(op)) { 0.00 : ffff80001045adcc: tbz w0, #0, ffff80001045ae18 : if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 0.00 : ffff80001045add0: tst w0, #0x60000 0.00 : ffff80001045add4: b.ne ffff80001045b114 // b.any : WARN_ONCE(1, 0.00 : ffff80001045add8: adrp x2, ffff800011ad8000 0.00 : ffff80001045addc: add x2, x2, #0x50a 0.00 : ffff80001045ade0: ldrb w0, [x2, #1] 0.00 : ffff80001045ade4: cbnz w0, ffff80001045ae18 0.00 : ffff80001045ade8: mov w3, #0x1 // #1 0.00 : ffff80001045adec: add x1, x29, #0x48 0.00 : ffff80001045adf0: strb w3, [x2, #1] 0.00 : ffff80001045adf4: mov x0, x19 0.00 : ffff80001045adf8: bl ffff800010472a70 0.00 : ffff80001045adfc: mov x1, x0 0.00 : ffff80001045ae00: ldr w2, [x22, #820] 0.00 : ffff80001045ae04: adrp x3, ffff8000111e9000 0.00 : ffff80001045ae08: add x0, x3, #0xa68 0.00 : ffff80001045ae0c: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff80001045ae10: brk #0x800 0.00 : ffff80001045ae14: nop 0.00 : ffff80001045ae18: ldr w0, [x19, #40] 0.00 : ffff80001045ae1c: lsr w0, w0, #9 : blk_partition_remap(): : if (bio_sectors(bio)) { 0.00 : ffff80001045ae20: cbnz w0, ffff80001045b0c8 : bio->bi_partno = 0; 0.00 : ffff80001045ae24: strb wzr, [x19, #27] : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff80001045ae28: bl ffff800010160d20 <__rcu_read_unlock> : generic_make_request_checks(): : if (op_is_flush(bio->bi_opf) && 0.00 : ffff80001045ae2c: ldr w0, [x19, #16] 0.00 : ffff80001045ae30: tst w0, #0x60000 0.00 : ffff80001045ae34: b.ne ffff80001045b034 // b.any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 2.28 : ffff80001045ae38: ldr x1, [x21, #104] : generic_make_request_checks(): : if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 0.00 : ffff80001045ae3c: tst w1, #0x10000 0.00 : ffff80001045ae40: b.ne ffff80001045ae4c // b.any : bio->bi_opf &= ~REQ_HIPRI; 4.70 : ffff80001045ae44: and w0, w0, #0xfdffffff 0.58 : ffff80001045ae48: str w0, [x19, #16] : switch (bio_op(bio)) { 0.00 : ffff80001045ae4c: and w0, w0, #0xff 0.00 : ffff80001045ae50: cmp w0, #0x7 0.00 : ffff80001045ae54: b.eq ffff80001045b168 // b.none 0.00 : ffff80001045ae58: b.hi ffff80001045b068 // b.pmore 0.00 : ffff80001045ae5c: cmp w0, #0x5 0.00 : ffff80001045ae60: b.eq ffff80001045b158 // b.none 0.00 : ffff80001045ae64: b.hi ffff80001045b100 // b.pmore 1.72 : ffff80001045ae68: cmp w0, #0x3 0.00 : ffff80001045ae6c: b.ne ffff80001045ae80 // b.any : test_bit(): 0.00 : ffff80001045ae70: ldr x0, [x21, #104] : generic_make_request_checks(): : if (!blk_queue_discard(q)) 0.00 : ffff80001045ae74: tst w0, #0x100 0.00 : ffff80001045ae78: b.eq ffff80001045b090 // b.none 0.00 : ffff80001045ae7c: nop : create_io_context(GFP_ATOMIC, q->node); 0.00 : ffff80001045ae80: ldr w2, [x21, #1156] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.58 : ffff80001045ae84: mrs x1, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001045ae88: and w0, w1, #0x80 : create_io_context(): : * Note that this function can't be called with IRQ disabled because : * task_lock which protects %current->io_context is IRQ-unsafe. : */ : static inline struct io_context *create_io_context(gfp_t gfp_mask, int node) : { : WARN_ON_ONCE(irqs_disabled()); 0.00 : ffff80001045ae8c: cbnz w0, ffff80001045b190 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 1.17 : ffff80001045ae90: mrs x0, sp_el0 : create_io_context(): : if (unlikely(!current->io_context)) 2.34 : ffff80001045ae94: ldr x1, [x0, #1864] 0.00 : ffff80001045ae98: cbz x1, ffff80001045b184 : rcu_read_lock(): : __rcu_read_lock(); 2.28 : ffff80001045ae9c: bl ffff80001015c1b0 <__rcu_read_lock> : blkcg_bio_issue_check(): : struct blkcg_gq *blkg; : bool throtl = false; : : rcu_read_lock(); : : if (!bio->bi_blkg) { 0.00 : ffff80001045aea0: ldr x3, [x19, #72] 0.00 : ffff80001045aea4: cbz x3, ffff80001045b12c : : if (!throtl) { : struct blkg_iostat_set *bis; : int rwd, cpu; : : if (op_is_discard(bio->bi_opf)) 2.25 : ffff80001045aea8: ldr w0, [x19, #16] : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 0.00 : ffff80001045aeac: mov w4, #0x2 // #2 : get_current(): 0.00 : ffff80001045aeb0: mrs x2, sp_el0 : op_is_discard(): : (op & (REQ_SYNC | REQ_FUA | REQ_PREFLUSH)); : } : : static inline bool op_is_discard(unsigned int op) : { : return (op & REQ_OP_MASK) == REQ_OP_DISCARD; 0.00 : ffff80001045aeb4: and w5, w0, #0xff : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001045aeb8: ldr w1, [x2, #16] : op_is_write(): : return (op & 1); 0.00 : ffff80001045aebc: cmp w5, #0x3 0.00 : ffff80001045aec0: and w0, w0, #0x1 : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001045aec4: add w1, w1, #0x1 : op_is_write(): 1.15 : ffff80001045aec8: csel w0, w0, w4, ne // ne = any : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001045aecc: str w1, [x2, #16] : blkcg_bio_issue_check(): : else if (op_is_write(bio->bi_opf)) : rwd = BLKG_IOSTAT_WRITE; : else : rwd = BLKG_IOSTAT_READ; : : cpu = get_cpu(); 0.00 : ffff80001045aed0: adrp x1, ffff80001151d000 0.00 : ffff80001045aed4: add x1, x1, #0x18 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 1.17 : ffff80001045aed8: mrs x2, tpidr_el1 : blkcg_bio_issue_check(): 0.00 : ffff80001045aedc: ldr w1, [x1, x2] : bis = per_cpu_ptr(blkg->iostat_cpu, cpu); 0.00 : ffff80001045aee0: adrp x4, ffff800011909000 0.00 : ffff80001045aee4: add x4, x4, #0x928 : bio_flagged(): : atomic_set(&bio->__bi_cnt, count); : } : : static inline bool bio_flagged(struct bio *bio, unsigned int bit) : { : return (bio->bi_flags & (1U << bit)) != 0; 20.89 : ffff80001045aee8: ldrh w5, [x19, #20] : blkcg_bio_issue_check(): 0.00 : ffff80001045aeec: ldr x2, [x3, #128] 0.00 : ffff80001045aef0: sxtw x0, w0 0.00 : ffff80001045aef4: ldr x4, [x4, w1, sxtw #3] 0.00 : ffff80001045aef8: add x2, x2, x4 : /* : * If the bio is flagged with BIO_QUEUE_ENTERED it means this : * is a split bio and we would have already accounted for the : * size of the bio. : */ : if (!bio_flagged(bio, BIO_QUEUE_ENTERED)) 0.00 : ffff80001045aefc: tbnz w5, #11, ffff80001045af10 : bis->cur.bytes[rwd] += bio->bi_iter.bi_size; 5.24 : ffff80001045af00: ldr x4, [x2, x0, lsl #3] 13.63 : ffff80001045af04: ldr w5, [x19, #40] 0.00 : ffff80001045af08: add x4, x4, x5 0.00 : ffff80001045af0c: str x4, [x2, x0, lsl #3] 0.00 : ffff80001045af10: add x0, x2, x0, lsl #3 : bis->cur.ios[rwd]++; 1.17 : ffff80001045af14: ldr x2, [x0, #24] 0.00 : ffff80001045af18: add x2, x2, #0x1 0.58 : ffff80001045af1c: str x2, [x0, #24] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001045af20: nop : blkcg_bio_issue_check(): : : u64_stats_update_end(&bis->sync); : if (cgroup_subsys_on_dfl(io_cgrp_subsys)) : cgroup_rstat_updated(blkg->blkcg->css.cgroup, cpu); 0.00 : ffff80001045af24: ldr x0, [x3, #40] 0.00 : ffff80001045af28: ldr x0, [x0] 0.00 : ffff80001045af2c: bl ffff800010196d80 : get_current(): 1.17 : ffff80001045af30: mrs x1, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045af34: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045af38: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001045af3c: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045af40: cbz x0, ffff80001045b060 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045af44: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045af48: cbz x0, ffff80001045b060 : blkcg_bio_issue_init(): : bio_issue_init(&bio->bi_issue, bio_sectors(bio)); 0.59 : ffff80001045af4c: ldr w21, [x19, #40] : bio_issue_init(): : issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | 0.00 : ffff80001045af50: ldr x22, [x19, #80] : ktime_get_ns(): : return ktime_mono_to_any(mono, TK_OFFS_REAL); : } : : static inline u64 ktime_get_ns(void) : { : return ktime_to_ns(ktime_get()); 0.00 : ffff80001045af54: bl ffff80001016c938 : bio_issue_init(): : (ktime_get_ns() & BIO_ISSUE_TIME_MASK) | 0.00 : ffff80001045af58: and x0, x0, #0x7ffffffffffff : blkcg_bio_issue_init(): 0.00 : ffff80001045af5c: lsr w21, w21, #9 : bio_issue_init(): : issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | 0.00 : ffff80001045af60: and x22, x22, #0x8000000000000000 : ((u64)size << BIO_ISSUE_SIZE_SHIFT)); 0.00 : ffff80001045af64: ubfiz x21, x21, #51, #12 : (ktime_get_ns() & BIO_ISSUE_TIME_MASK) | 0.00 : ffff80001045af68: orr x21, x21, x22 0.00 : ffff80001045af6c: orr x0, x21, x0 : issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | 0.00 : ffff80001045af70: str x0, [x19, #80] : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff80001045af74: bl ffff800010160d20 <__rcu_read_unlock> : generic_make_request_checks(): : if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) { 0.00 : ffff80001045af78: ldrh w1, [x19, #20] : return true; 0.00 : ffff80001045af7c: mov w0, #0x1 // #1 : if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) { 0.00 : ffff80001045af80: tbnz w1, #10, ffff80001045b0c0 : bio_set_flag(): : } : : static inline void bio_set_flag(struct bio *bio, unsigned int bit) : { : bio->bi_flags |= (1U << bit); 1.16 : ffff80001045af84: orr w1, w1, #0x400 0.00 : ffff80001045af88: strh w1, [x19, #20] 0.00 : ffff80001045af8c: ldr x23, [x29, #48] 0.00 : ffff80001045af90: b ffff80001045ad70 : bio_check_ro(): : if (part->policy && op_is_write(op)) { 0.58 : ffff80001045af94: ldr w2, [x22, #888] : generic_make_request_checks(): : int nr_sectors = bio_sectors(bio); 0.00 : ffff80001045af98: mov w0, w23 : bio_check_ro(): : if (part->policy && op_is_write(op)) { 0.00 : ffff80001045af9c: cbz w2, ffff80001045b000 0.00 : ffff80001045afa0: tbz w1, #0, ffff80001045b000 : if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 0.00 : ffff80001045afa4: tst w1, #0x60000 0.00 : ffff80001045afa8: mov x0, #0x0 // #0 0.00 : ffff80001045afac: ccmp w23, #0x0, #0x0, ne // ne = any 0.00 : ffff80001045afb0: b.eq ffff80001045b000 // b.none : WARN_ONCE(1, 0.00 : ffff80001045afb4: adrp x2, ffff800011ad8000 0.00 : ffff80001045afb8: add x2, x2, #0x50a : generic_make_request_checks(): : int nr_sectors = bio_sectors(bio); 0.00 : ffff80001045afbc: mov w0, w23 : bio_check_ro(): : WARN_ONCE(1, 0.00 : ffff80001045afc0: ldrb w1, [x2, #1] 0.00 : ffff80001045afc4: cbnz w1, ffff80001045b000 0.00 : ffff80001045afc8: mov w3, #0x1 // #1 0.00 : ffff80001045afcc: add x1, x29, #0x48 0.00 : ffff80001045afd0: strb w3, [x2, #1] 0.00 : ffff80001045afd4: mov x0, x19 0.00 : ffff80001045afd8: bl ffff800010472a70 0.00 : ffff80001045afdc: mov x1, x0 0.00 : ffff80001045afe0: ldr w2, [x22, #892] 0.00 : ffff80001045afe4: adrp x3, ffff8000111e9000 0.00 : ffff80001045afe8: add x0, x3, #0xa68 0.00 : ffff80001045afec: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff80001045aff0: brk #0x800 0.00 : ffff80001045aff4: ldr w0, [x19, #40] 0.00 : ffff80001045aff8: ldr x22, [x19, #8] 0.00 : ffff80001045affc: lsr w0, w0, #9 : generic_make_request_checks(): : if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk)))) 4.69 : ffff80001045b000: ldr x1, [x22, #80] : bio_check_eod(): : if (nr_sectors && maxsector && 0.00 : ffff80001045b004: cmp w0, #0x0 0.00 : ffff80001045b008: ccmp x1, #0x0, #0x4, ne // ne = any 0.00 : ffff80001045b00c: b.eq ffff80001045b028 // b.none 9.81 : ffff80001045b010: cmp x1, x0 0.00 : ffff80001045b014: b.cc ffff80001045b1dc // b.lo, b.ul, b.last : (nr_sectors > maxsector || 1.11 : ffff80001045b018: ldr x2, [x19, #32] : bio->bi_iter.bi_sector > maxsector - nr_sectors)) { 0.00 : ffff80001045b01c: sub x0, x1, x0 : (nr_sectors > maxsector || 0.58 : ffff80001045b020: cmp x2, x0 0.00 : ffff80001045b024: b.hi ffff80001045b1dc // b.pmore : generic_make_request_checks(): : if (op_is_flush(bio->bi_opf) && 2.26 : ffff80001045b028: ldr w0, [x19, #16] 0.00 : ffff80001045b02c: tst w0, #0x60000 0.00 : ffff80001045b030: b.eq ffff80001045ae38 // b.none : test_bit(): 0.00 : ffff80001045b034: ldr x1, [x21, #104] : generic_make_request_checks(): 0.00 : ffff80001045b038: tst w1, #0x20000 0.00 : ffff80001045b03c: b.ne ffff80001045ae38 // b.any : bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA); 0.00 : ffff80001045b040: and w0, w0, #0xfff9ffff 0.00 : ffff80001045b044: str w0, [x19, #16] : if (!nr_sectors) { 0.00 : ffff80001045b048: cbnz w23, ffff80001045ae38 : status = BLK_STS_OK; 0.00 : ffff80001045b04c: mov w0, #0x0 // #0 0.00 : ffff80001045b050: ldr x23, [x29, #48] 0.00 : ffff80001045b054: b ffff80001045ad60 : status = BLK_STS_AGAIN; 0.00 : ffff80001045b058: mov w0, #0xc // #12 0.00 : ffff80001045b05c: b ffff80001045ad60 : blkcg_bio_issue_check(): : put_cpu(); 0.00 : ffff80001045b060: bl ffff800010cd2e50 0.00 : ffff80001045b064: b ffff80001045af4c : generic_make_request_checks(): : switch (bio_op(bio)) { 0.00 : ffff80001045b068: cmp w0, #0x9 0.00 : ffff80001045b06c: b.eq ffff80001045b14c // b.none 0.00 : ffff80001045b070: b.cs ffff80001045b0f8 // b.hs, b.nlast : blk_queue_is_zoned(): : return q->limits.zoned; : } : : static inline bool blk_queue_is_zoned(struct request_queue *q) : { : switch (blk_queue_zoned_model(q)) { 0.00 : ffff80001045b074: ldr w0, [x21, #1140] 0.00 : ffff80001045b078: sub w0, w0, #0x1 0.00 : ffff80001045b07c: cmp w0, #0x1 0.00 : ffff80001045b080: b.hi ffff80001045b090 // b.pmore : test_bit(): 0.00 : ffff80001045b084: ldr x0, [x21, #104] : generic_make_request_checks(): : if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q)) 0.00 : ffff80001045b088: tst w0, #0x4000000 0.00 : ffff80001045b08c: b.ne ffff80001045ae80 // b.any : status = BLK_STS_NOTSUPP; 0.00 : ffff80001045b090: mov w0, #0x1 // #1 0.00 : ffff80001045b094: ldr x23, [x29, #48] 0.00 : ffff80001045b098: b ffff80001045ad60 : printk(KERN_ERR 0.00 : ffff80001045b09c: add x1, x29, #0x48 0.00 : ffff80001045b0a0: bl ffff800010472a70 0.00 : ffff80001045b0a4: ldr x2, [x19, #32] 0.00 : ffff80001045b0a8: mov x1, x0 0.00 : ffff80001045b0ac: adrp x0, ffff8000111e9000 0.00 : ffff80001045b0b0: add x0, x0, #0xa18 0.00 : ffff80001045b0b4: bl ffff80001014a85c : blk_status_t status = BLK_STS_IOERR; 0.00 : ffff80001045b0b8: mov w0, #0xa // #10 : goto end_io; 0.00 : ffff80001045b0bc: b ffff80001045ad60 0.00 : ffff80001045b0c0: ldr x23, [x29, #48] 0.00 : ffff80001045b0c4: b ffff80001045ad70 : blk_partition_remap(): : if (bio_check_eod(bio, part_nr_sects_read(p))) 0.00 : ffff80001045b0c8: ldr x1, [x22, #8] : bio_check_eod(): : if (nr_sectors && maxsector && 0.00 : ffff80001045b0cc: cbz x1, ffff80001045b124 0.00 : ffff80001045b0d0: cmp x1, x0 0.00 : ffff80001045b0d4: b.cc ffff80001045b1c4 // b.lo, b.ul, b.last : bio->bi_iter.bi_sector > maxsector - nr_sectors)) { 0.00 : ffff80001045b0d8: ldr x2, [x19, #32] 0.00 : ffff80001045b0dc: sub x0, x1, x0 : (nr_sectors > maxsector || 0.00 : ffff80001045b0e0: cmp x2, x0 0.00 : ffff80001045b0e4: b.hi ffff80001045b1c4 // b.pmore : blk_partition_remap(): : bio->bi_iter.bi_sector += p->start_sect; 0.00 : ffff80001045b0e8: ldr x0, [x22] 0.00 : ffff80001045b0ec: add x0, x0, x2 0.00 : ffff80001045b0f0: str x0, [x19, #32] 0.00 : ffff80001045b0f4: b ffff80001045ae24 : generic_make_request_checks(): : switch (bio_op(bio)) { 0.00 : ffff80001045b0f8: cmp w0, #0xc 0.00 : ffff80001045b0fc: b.hi ffff80001045ae80 // b.pmore : blk_queue_is_zoned(): 0.00 : ffff80001045b100: ldr w0, [x21, #1140] 0.00 : ffff80001045b104: sub w0, w0, #0x1 0.00 : ffff80001045b108: cmp w0, #0x1 0.00 : ffff80001045b10c: b.hi ffff80001045b090 // b.pmore 0.00 : ffff80001045b110: b ffff80001045ae80 : bio_check_ro(): : if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 0.00 : ffff80001045b114: ldr w0, [x19, #40] 0.00 : ffff80001045b118: negs wzr, w0, lsr #9 0.00 : ffff80001045b11c: b.eq ffff80001045ae24 // b.none 0.00 : ffff80001045b120: b ffff80001045add8 0.00 : ffff80001045b124: ldr x2, [x19, #32] 0.00 : ffff80001045b128: b ffff80001045b0e8 : blkcg_bio_issue_check(): : WARN_ONCE(1, 0.00 : ffff80001045b12c: adrp x2, ffff800011ad8000 0.00 : ffff80001045b130: add x2, x2, #0x50a 0.00 : ffff80001045b134: ldrb w0, [x2, #2] 0.00 : ffff80001045b138: cbz w0, ffff80001045b198 : bio_associate_blkg(bio); 0.00 : ffff80001045b13c: mov x0, x19 0.00 : ffff80001045b140: bl ffff800010456970 0.00 : ffff80001045b144: ldr x3, [x19, #72] 0.00 : ffff80001045b148: b ffff80001045aea8 : generic_make_request_checks(): : if (!q->limits.max_write_zeroes_sectors) 0.00 : ffff80001045b14c: ldr w0, [x21, #1116] 0.00 : ffff80001045b150: cbz w0, ffff80001045b090 0.00 : ffff80001045b154: b ffff80001045ae80 : test_bit(): 0.00 : ffff80001045b158: ldr x0, [x21, #104] : generic_make_request_checks(): : if (!blk_queue_secure_erase(q)) 0.00 : ffff80001045b15c: tst w0, #0x800 0.00 : ffff80001045b160: b.eq ffff80001045b090 // b.none 0.00 : ffff80001045b164: b ffff80001045ae80 : if (!q->limits.max_write_same_sectors) 0.00 : ffff80001045b168: ldr w0, [x21, #1112] 0.00 : ffff80001045b16c: cbz w0, ffff80001045b090 0.00 : ffff80001045b170: b ffff80001045ae80 : rcu_read_unlock(): 0.00 : ffff80001045b174: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff80001045b178: ldr x23, [x29, #48] : generic_make_request_checks(): : blk_status_t status = BLK_STS_IOERR; 0.00 : ffff80001045b17c: mov w0, #0xa // #10 0.00 : ffff80001045b180: b ffff80001045ad60 : create_io_context(): : create_task_io_context(current, gfp_mask, node); 0.00 : ffff80001045b184: mov w1, #0xa20 // #2592 0.00 : ffff80001045b188: bl ffff800010460168 0.00 : ffff80001045b18c: b ffff80001045ae9c : WARN_ON_ONCE(irqs_disabled()); 0.00 : ffff80001045b190: brk #0x800 0.00 : ffff80001045b194: b ffff80001045ae90 : blkcg_bio_issue_check(): : WARN_ONCE(1, 0.00 : ffff80001045b198: mov w3, #0x1 // #1 0.00 : ffff80001045b19c: add x1, x29, #0x48 0.00 : ffff80001045b1a0: strb w3, [x2, #2] 0.00 : ffff80001045b1a4: mov x0, x19 0.00 : ffff80001045b1a8: bl ffff800010472a70 0.00 : ffff80001045b1ac: mov x1, x0 0.00 : ffff80001045b1b0: adrp x2, ffff8000111e9000 0.00 : ffff80001045b1b4: add x0, x2, #0xab8 0.00 : ffff80001045b1b8: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff80001045b1bc: brk #0x800 0.00 : ffff80001045b1c0: b ffff80001045b13c : bio_check_eod(): : handle_bad_sector(bio, maxsector); 0.00 : ffff80001045b1c4: mov x0, x19 0.00 : ffff80001045b1c8: bl ffff80001045a400 : rcu_read_unlock(): 0.00 : ffff80001045b1cc: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff80001045b1d0: ldr x23, [x29, #48] : generic_make_request_checks(): : blk_status_t status = BLK_STS_IOERR; 0.00 : ffff80001045b1d4: mov w0, #0xa // #10 0.00 : ffff80001045b1d8: b ffff80001045ad60 : bio_check_eod(): : handle_bad_sector(bio, maxsector); 0.00 : ffff80001045b1dc: mov x0, x19 0.00 : ffff80001045b1e0: bl ffff80001045a400 : generic_make_request_checks(): : blk_status_t status = BLK_STS_IOERR; 0.00 : ffff80001045b1e4: mov w0, #0xa // #10 0.00 : ffff80001045b1e8: ldr x23, [x29, #48] 0.00 : ffff80001045b1ec: b ffff80001045ad60 0.00 : ffff80001045b1f0: str x23, [x29, #48] : } 0.00 : ffff80001045b1f4: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (77 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017bb98 : : tick_nohz_idle_stop_tick(): : * tick_nohz_idle_stop_tick - stop the idle tick from the idle task : * : * When the next event is more than a tick into the future, stop the idle tick : */ : void tick_nohz_idle_stop_tick(void) : { 0.00 : ffff80001017bb98: stp x29, x30, [sp, #-80]! : __tick_nohz_idle_stop_tick(): : int cpu = smp_processor_id(); 0.00 : ffff80001017bb9c: adrp x0, ffff80001151d000 0.00 : ffff80001017bba0: add x0, x0, #0x18 : tick_nohz_idle_stop_tick(): : { 0.00 : ffff80001017bba4: mov x29, sp 0.00 : ffff80001017bba8: stp x19, x20, [sp, #16] : __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched)); 0.00 : ffff80001017bbac: adrp x19, ffff800011525000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001017bbb0: mrs x1, tpidr_el1 : tick_nohz_idle_stop_tick(): 13.05 : ffff80001017bbb4: add x19, x19, #0xed0 : { 0.00 : ffff80001017bbb8: str x22, [sp, #40] : __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched)); 0.00 : ffff80001017bbbc: add x19, x19, x1 : __tick_nohz_idle_stop_tick(): : int cpu = smp_processor_id(); 0.00 : ffff80001017bbc0: ldr w20, [x0, x1] : if (ts->timer_expires_base) 0.00 : ffff80001017bbc4: ldr x2, [x19, #176] 0.00 : ffff80001017bbc8: cbz x2, ffff80001017bcc0 : expires = ts->timer_expires; 0.00 : ffff80001017bbcc: ldr x22, [x19, #168] : ts->idle_calls++; 0.00 : ffff80001017bbd0: ldr x0, [x19, #104] : if (expires > 0LL) { 0.00 : ffff80001017bbd4: cmp x22, #0x0 : ts->idle_calls++; 0.00 : ffff80001017bbd8: add x0, x0, #0x1 2.71 : ffff80001017bbdc: str x0, [x19, #104] : if (expires > 0LL) { 0.00 : ffff80001017bbe0: b.gt ffff80001017bbf8 : tick_nohz_retain_tick(): : ts->timer_expires_base = 0; 0.00 : ffff80001017bbe4: str xzr, [x19, #176] : tick_nohz_idle_stop_tick(): : } 0.00 : ffff80001017bbe8: ldp x19, x20, [sp, #16] 0.00 : ffff80001017bbec: ldr x22, [sp, #40] 0.00 : ffff80001017bbf0: ldp x29, x30, [sp], #80 0.00 : ffff80001017bbf4: ret 0.00 : ffff80001017bbf8: str x21, [x29, #32] : tick_nohz_stop_tick(): : if (cpu == tick_do_timer_cpu) { 0.00 : ffff80001017bbfc: adrp x4, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001017bc00: stp x23, x24, [x29, #48] : struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); 0.00 : ffff80001017bc04: adrp x0, ffff800011525000 : __my_cpu_offset(): 0.00 : ffff80001017bc08: mrs x5, tpidr_el1 : __tick_nohz_idle_stop_tick(): : int was_stopped = ts->tick_stopped; 6.42 : ffff80001017bc0c: ldrb w2, [x19, #76] : tick_nohz_stop_tick(): : struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); 0.00 : ffff80001017bc10: add x0, x0, #0xec0 : if (cpu == tick_do_timer_cpu) { 0.00 : ffff80001017bc14: ldr w3, [x4, #712] : u64 basemono = ts->timer_expires_base; 0.00 : ffff80001017bc18: ldp x21, x24, [x19, #168] : if (cpu == tick_do_timer_cpu) { 0.00 : ffff80001017bc1c: cmp w20, w3 : struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev); 2.81 : ffff80001017bc20: ldr x0, [x0, x5] : __tick_nohz_idle_stop_tick(): : int was_stopped = ts->tick_stopped; 0.00 : ffff80001017bc24: ubfx x23, x2, #1, #1 : tick_nohz_stop_tick(): : ts->timer_expires_base = 0; 7.77 : ffff80001017bc28: str xzr, [x19, #176] : if (cpu == tick_do_timer_cpu) { 0.00 : ffff80001017bc2c: b.eq ffff80001017bd2c // b.none : } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) { 0.00 : ffff80001017bc30: cmn w3, #0x1 0.00 : ffff80001017bc34: b.eq ffff80001017bc40 // b.none : ts->do_timer_last = 0; 0.00 : ffff80001017bc38: and w2, w2, #0xfffffff7 1.33 : ffff80001017bc3c: strb w2, [x19, #76] : if (ts->tick_stopped && (expires == ts->next_tick)) { 0.00 : ffff80001017bc40: tbnz w2, #1, ffff80001017bcec : calc_load_nohz_start(); 9.13 : ffff80001017bc44: bl ffff80001011adc8 : quiet_vmstat(); 7.79 : ffff80001017bc48: bl ffff8000101f75f8 : ts->tick_stopped = 1; 2.68 : ffff80001017bc4c: ldrb w0, [x19, #76] : ts->last_tick = hrtimer_get_expires(&ts->sched_timer); 0.00 : ffff80001017bc50: ldr x1, [x19, #24] : ts->tick_stopped = 1; 0.00 : ffff80001017bc54: orr w0, w0, #0x2 0.00 : ffff80001017bc58: strb w0, [x19, #76] : ts->last_tick = hrtimer_get_expires(&ts->sched_timer); 2.66 : ffff80001017bc5c: str x1, [x19, #80] : ts->next_tick = tick; 0.00 : ffff80001017bc60: str x21, [x19, #88] : if (unlikely(expires == KTIME_MAX)) { 0.00 : ffff80001017bc64: mov x1, #0x7fffffffffffffff // #9223372036854775807 0.00 : ffff80001017bc68: ldr w0, [x19, #72] 0.00 : ffff80001017bc6c: cmp x21, x1 0.00 : ffff80001017bc70: b.eq ffff80001017bd78 // b.none : if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { 2.72 : ffff80001017bc74: cmp w0, #0x2 0.00 : ffff80001017bc78: b.eq ffff80001017bd44 // b.none : hrtimer_set_expires(): : } ____cacheline_aligned; : : static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) : { : timer->node.expires = time; : timer->_softexpires = time; 0.00 : ffff80001017bc7c: stp x21, x21, [x19, #24] : tick_nohz_stop_tick(): : tick_program_event(tick, 1); 0.00 : ffff80001017bc80: mov w1, #0x1 // #1 0.00 : ffff80001017bc84: mov x0, x21 0.00 : ffff80001017bc88: bl ffff80001017afa8 : __tick_nohz_idle_stop_tick(): : ts->idle_sleeps++; 0.00 : ffff80001017bc8c: ldr x0, [x19, #112] : ts->idle_expires = expires; 0.00 : ffff80001017bc90: str x22, [x19, #192] : ts->idle_sleeps++; 0.00 : ffff80001017bc94: add x0, x0, #0x1 0.00 : ffff80001017bc98: str x0, [x19, #112] : if (!was_stopped && ts->tick_stopped) { 0.00 : ffff80001017bc9c: cbnz w23, ffff80001017bca8 1.31 : ffff80001017bca0: ldrb w0, [x19, #76] 0.00 : ffff80001017bca4: tbnz w0, #1, ffff80001017bd5c 0.00 : ffff80001017bca8: ldr x21, [x29, #32] 0.00 : ffff80001017bcac: ldp x23, x24, [x29, #48] : tick_nohz_idle_stop_tick(): : } 0.00 : ffff80001017bcb0: ldp x19, x20, [sp, #16] 0.00 : ffff80001017bcb4: ldr x22, [sp, #40] 0.00 : ffff80001017bcb8: ldp x29, x30, [sp], #80 0.00 : ffff80001017bcbc: ret : __tick_nohz_idle_stop_tick(): : else if (can_stop_idle_tick(cpu, ts)) 3.92 : ffff80001017bcc0: ldr w1, [x19, #72] 0.00 : ffff80001017bcc4: add x2, x19, #0x58 0.00 : ffff80001017bcc8: mov w0, w20 0.00 : ffff80001017bccc: bl ffff80001017b8b8 7.72 : ffff80001017bcd0: tst w0, #0xff 0.00 : ffff80001017bcd4: b.eq ffff80001017bbe8 // b.none : expires = tick_nohz_next_event(ts, cpu); 2.61 : ffff80001017bcd8: mov w1, w20 0.00 : ffff80001017bcdc: mov x0, x19 0.00 : ffff80001017bce0: bl ffff80001017b9b8 0.00 : ffff80001017bce4: mov x22, x0 0.00 : ffff80001017bce8: b ffff80001017bbd0 : tick_nohz_stop_tick(): : if (ts->tick_stopped && (expires == ts->next_tick)) { 0.00 : ffff80001017bcec: ldr x2, [x19, #88] 0.00 : ffff80001017bcf0: cmp x21, x2 0.00 : ffff80001017bcf4: b.ne ffff80001017bc60 // b.any : if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer)) 0.00 : ffff80001017bcf8: mov x2, #0x7fffffffffffffff // #9223372036854775807 0.00 : ffff80001017bcfc: cmp x21, x2 0.00 : ffff80001017bd00: b.eq ffff80001017bc8c // b.none 0.00 : ffff80001017bd04: ldr x2, [x19, #24] 0.00 : ffff80001017bd08: cmp x21, x2 0.00 : ffff80001017bd0c: b.eq ffff80001017bc8c // b.none : WARN_ON_ONCE(1); 0.00 : ffff80001017bd10: brk #0x800 : printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n", 0.00 : ffff80001017bd14: adrp x4, ffff800011ad8000 0.00 : ffff80001017bd18: ldrb w2, [x4, #1237] 0.00 : ffff80001017bd1c: cbz w2, ffff80001017bd8c : if (!ts->tick_stopped) { 0.00 : ffff80001017bd20: ldrb w0, [x19, #76] 0.00 : ffff80001017bd24: tbnz w0, #1, ffff80001017bc60 0.00 : ffff80001017bd28: b ffff80001017bc44 : ts->do_timer_last = 1; 0.00 : ffff80001017bd2c: orr w2, w2, #0x8 : tick_do_timer_cpu = TICK_DO_TIMER_NONE; 0.00 : ffff80001017bd30: mov w3, #0xffffffff // #-1 : ts->do_timer_last = 1; 0.00 : ffff80001017bd34: strb w2, [x19, #76] : tick_do_timer_cpu = TICK_DO_TIMER_NONE; 0.00 : ffff80001017bd38: str w3, [x4, #712] : if (ts->tick_stopped && (expires == ts->next_tick)) { 0.00 : ffff80001017bd3c: tbnz w2, #1, ffff80001017bcec 0.00 : ffff80001017bd40: b ffff80001017bc44 : hrtimer_start(): : * softirq based mode is considered for debug purpose only! : */ : static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim, : const enum hrtimer_mode mode) : { : hrtimer_start_range_ns(timer, tim, 0, mode); 17.36 : ffff80001017bd44: mov w3, #0xa // #10 0.00 : ffff80001017bd48: mov x2, #0x0 // #0 0.00 : ffff80001017bd4c: mov x1, x21 0.00 : ffff80001017bd50: mov x0, x19 0.00 : ffff80001017bd54: bl ffff80001016b750 0.00 : ffff80001017bd58: b ffff80001017bc8c : __tick_nohz_idle_stop_tick(): : ts->idle_jiffies = ts->last_jiffies; 0.00 : ffff80001017bd5c: ldr x1, [x19, #160] : nohz_balance_enter_idle(cpu); 0.00 : ffff80001017bd60: mov w0, w20 : ts->idle_jiffies = ts->last_jiffies; 1.32 : ffff80001017bd64: str x1, [x19, #96] : nohz_balance_enter_idle(cpu); 0.00 : ffff80001017bd68: bl ffff800010127460 0.00 : ffff80001017bd6c: ldr x21, [x29, #32] 6.69 : ffff80001017bd70: ldp x23, x24, [x29, #48] 0.00 : ffff80001017bd74: b ffff80001017bbe8 : tick_nohz_stop_tick(): : if (ts->nohz_mode == NOHZ_MODE_HIGHRES) 0.00 : ffff80001017bd78: cmp w0, #0x2 0.00 : ffff80001017bd7c: b.ne ffff80001017bc8c // b.any : hrtimer_cancel(&ts->sched_timer); 0.00 : ffff80001017bd80: mov x0, x19 0.00 : ffff80001017bd84: bl ffff80001016bad0 0.00 : ffff80001017bd88: b ffff80001017bc8c 0.00 : ffff80001017bd8c: stp x25, x26, [x29, #64] : printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n", 0.00 : ffff80001017bd90: mov w5, #0x1 // #1 0.00 : ffff80001017bd94: strb w5, [x4, #1237] 0.00 : ffff80001017bd98: ldr x26, [x0, #24] 0.00 : ffff80001017bd9c: mov x0, x19 0.00 : ffff80001017bda0: ldr x25, [x19, #88] 0.00 : ffff80001017bda4: bl ffff80001016aeb8 0.00 : ffff80001017bda8: ldr x5, [x19, #24] 0.00 : ffff80001017bdac: adrp x6, ffff8000111c9000 0.00 : ffff80001017bdb0: and w4, w0, #0xff 0.00 : ffff80001017bdb4: mov x3, x26 0.00 : ffff80001017bdb8: mov x2, x25 0.00 : ffff80001017bdbc: mov x1, x24 0.00 : ffff80001017bdc0: add x0, x6, #0x830 0.00 : ffff80001017bdc4: bl ffff80001014a85c 0.00 : ffff80001017bdc8: ldp x25, x26, [x29, #64] 0.00 : ffff80001017bdcc: b ffff80001017bd20 Percent | Source code & Disassembly of vmlinux for cycles (195 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001045c3d8 : : blk_do_io_stat(): : * a) it's attached to a gendisk, and : * b) the queue had IO stats enabled when this request was started : */ : static inline bool blk_do_io_stat(struct request *rq) : { : return rq->rq_disk && (rq->rq_flags & RQF_IO_STAT); 9.77 : ffff80001045c3d8: ldr x3, [x0, #160] : blk_account_io_start(): : } : : void blk_account_io_start(struct request *rq, bool new_io) : { : struct hd_struct *part; : int rw = rq_data_dir(rq); 9.33 : ffff80001045c3dc: ldr w2, [x0, #24] : blk_do_io_stat(): 0.00 : ffff80001045c3e0: cbz x3, ffff80001045c4e0 6.18 : ffff80001045c3e4: ldr w3, [x0, #28] 0.00 : ffff80001045c3e8: tbz w3, #13, ffff80001045c4e4 : blk_account_io_start(): : { 4.67 : ffff80001045c3ec: stp x29, x30, [sp, #-48]! 0.00 : ffff80001045c3f0: mov x29, sp 0.52 : ffff80001045c3f4: stp x19, x20, [sp, #16] 0.00 : ffff80001045c3f8: mov x19, x0 0.52 : ffff80001045c3fc: stp x21, x22, [sp, #32] 0.53 : ffff80001045c400: and w20, w1, #0xff 0.00 : ffff80001045c404: and w22, w2, #0x1 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001045c408: bl ffff80001015c1b0 <__rcu_read_lock> : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001045c40c: mrs x21, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001045c410: ldr w0, [x21, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001045c414: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.53 : ffff80001045c418: str w0, [x21, #16] : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff80001045c41c: mov x5, sp : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001045c420: mrs x2, tpidr_el1 : blk_account_io_start(): : if (!blk_do_io_stat(rq)) : return; : : part_stat_lock(); : : if (!new_io) { 0.00 : ffff80001045c424: cbnz w20, ffff80001045c4f4 : part = rq->part; : part_stat_inc(part, merges[rw]); 0.00 : ffff80001045c428: adrp x1, ffff80001151d000 0.00 : ffff80001045c42c: add x0, x1, #0x18 0.00 : ffff80001045c430: ldrsw x6, [x0, x2] 0.00 : ffff80001045c434: adrp x4, ffff800011909000 0.00 : ffff80001045c438: add x3, x4, #0x928 : part = rq->part; 0.00 : ffff80001045c43c: ldr x20, [x19, #168] 0.00 : ffff80001045c440: mov w2, w22 : part_stat_inc(part, merges[rw]); 0.00 : ffff80001045c444: ldr x3, [x3, x6, lsl #3] 0.00 : ffff80001045c448: lsl x2, x2, #3 0.00 : ffff80001045c44c: ldr x0, [x20, #840] 0.00 : ffff80001045c450: add x0, x0, x3 0.00 : ffff80001045c454: add x0, x0, x2 0.00 : ffff80001045c458: ldr x3, [x0, #96] 0.00 : ffff80001045c45c: add x3, x3, #0x1 0.00 : ffff80001045c460: str x3, [x0, #96] 0.00 : ffff80001045c464: ldr w0, [x20, #820] 0.00 : ffff80001045c468: cbz w0, ffff80001045c4a4 : part_to_disk(): : struct lockdep_map lockdep_map; : }; : : static inline struct gendisk *part_to_disk(struct hd_struct *part) : { : if (likely(part)) { 0.00 : ffff80001045c46c: cbz x20, ffff80001045c57c : if (part->partno) : return dev_to_disk(part_to_dev(part)->parent); 0.00 : ffff80001045c470: ldr x0, [x20, #104] 0.00 : ffff80001045c474: sub x0, x0, #0x70 : __my_cpu_offset(): 0.00 : ffff80001045c478: mrs x3, tpidr_el1 : blk_account_io_start(): 0.00 : ffff80001045c47c: add x1, x1, #0x18 0.00 : ffff80001045c480: add x4, x4, #0x928 0.00 : ffff80001045c484: ldrsw x1, [x1, x3] 0.00 : ffff80001045c488: ldr x0, [x0, #912] 0.00 : ffff80001045c48c: ldr x1, [x4, x1, lsl #3] 0.00 : ffff80001045c490: add x0, x0, x1 0.00 : ffff80001045c494: add x0, x0, x2 0.00 : ffff80001045c498: ldr x1, [x0, #96] 0.00 : ffff80001045c49c: add x1, x1, #0x1 0.00 : ffff80001045c4a0: str x1, [x0, #96] : } : part_inc_in_flight(rq->q, part, rw); : rq->part = part; : } : : update_io_ticks(part, jiffies); 1.58 : ffff80001045c4a4: adrp x1, ffff800011907000 0.00 : ffff80001045c4a8: mov x0, x20 0.00 : ffff80001045c4ac: ldr x1, [x1, #2432] 0.00 : ffff80001045c4b0: bl ffff800010458638 : get_current(): 0.51 : ffff80001045c4b4: mrs x1, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 6.66 : ffff80001045c4b8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045c4bc: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 13.16 : ffff80001045c4c0: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045c4c4: cbnz x0, ffff80001045c4e8 : blk_account_io_start(): : : part_stat_unlock(); 0.00 : ffff80001045c4c8: bl ffff800010cd2e50 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 1.55 : ffff80001045c4cc: bl ffff800010160d20 <__rcu_read_unlock> : blk_account_io_start(): : } 0.96 : ffff80001045c4d0: ldp x19, x20, [sp, #16] 0.47 : ffff80001045c4d4: ldp x21, x22, [sp, #32] 0.52 : ffff80001045c4d8: ldp x29, x30, [sp], #48 0.00 : ffff80001045c4dc: ret 0.00 : ffff80001045c4e0: ret 0.00 : ffff80001045c4e4: ret : __read_once_size(): : __READ_ONCE_SIZE; 2.08 : ffff80001045c4e8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045c4ec: cbz x0, ffff80001045c4c8 8.15 : ffff80001045c4f0: b ffff80001045c4cc : blk_account_io_start(): : part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); 0.00 : ffff80001045c4f4: ldr x1, [x19, #48] 3.05 : ffff80001045c4f8: ldr x0, [x19, #160] 0.00 : ffff80001045c4fc: bl ffff80001046f0e8 1.52 : ffff80001045c500: mov x20, x0 : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff80001045c504: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 1.04 : ffff80001045c508: ldr x0, [x20, #856] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001045c50c: tst x0, #0x3 0.00 : ffff80001045c510: b.ne ffff80001045c584 // b.any : __read_once_size(): 0.52 : ffff80001045c514: ldr w1, [x21, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff80001045c518: add w1, w1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 2.59 : ffff80001045c51c: str w1, [x21, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045c520: mov x2, #0x1 // #1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.51 : ffff80001045c524: mrs x1, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045c528: add x0, x0, x1 0.00 : ffff80001045c52c: ldxr x4, [x0] 10.76 : ffff80001045c530: add x4, x4, x2 0.00 : ffff80001045c534: stxr w3, x4, [x0] 0.00 : ffff80001045c538: cbnz w3, ffff80001045c52c : __read_once_size(): : __READ_ONCE_SIZE; 1.56 : ffff80001045c53c: ldr x0, [x21, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 1.05 : ffff80001045c540: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.50 : ffff80001045c544: str w0, [x21, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045c548: cbz x0, ffff80001045c570 : __read_once_size(): : __READ_ONCE_SIZE; 2.56 : ffff80001045c54c: ldr x0, [x21, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045c550: cbz x0, ffff80001045c570 : rcu_read_unlock(): : __rcu_read_unlock(); 5.16 : ffff80001045c554: bl ffff800010160d20 <__rcu_read_unlock> : blk_account_io_start(): : part_inc_in_flight(rq->q, part, rw); 0.47 : ffff80001045c558: ldr x0, [x19] 0.00 : ffff80001045c55c: mov w2, w22 0.00 : ffff80001045c560: mov x1, x20 0.00 : ffff80001045c564: bl ffff800010470618 : rq->part = part; 0.00 : ffff80001045c568: str x20, [x19, #168] 0.00 : ffff80001045c56c: b ffff80001045c4a4 : percpu_ref_tryget_live(): : bool ret = false; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) { : this_cpu_inc(*percpu_count); 0.00 : ffff80001045c570: bl ffff800010cd2e78 : rcu_read_unlock(): 0.00 : ffff80001045c574: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff80001045c578: b ffff80001045c558 : part_to_disk(): : else : return dev_to_disk(part_to_dev(part)); : } : return NULL; 0.00 : ffff80001045c57c: mov x0, #0x0 // #0 0.00 : ffff80001045c580: b ffff80001045c478 : percpu_ref_tryget_live(): : ret = true; : } else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) { 0.00 : ffff80001045c584: add x3, x20, #0x350 0.00 : ffff80001045c588: ldr x0, [x3, #8] 0.00 : ffff80001045c58c: tbz w0, #1, ffff80001045c5f8 : rcu_read_unlock(): 0.00 : ffff80001045c590: bl ffff800010160d20 <__rcu_read_unlock> : blk_account_io_start(): : part = &rq->rq_disk->part0; 0.00 : ffff80001045c594: ldr x21, [x19, #160] : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff80001045c598: bl ffff80001015c1b0 <__rcu_read_lock> : blk_account_io_start(): 0.00 : ffff80001045c59c: add x20, x21, #0x48 : __read_once_size(): 0.00 : ffff80001045c5a0: ldr x0, [x21, #928] : __ref_is_percpu(): : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001045c5a4: tst x0, #0x3 0.00 : ffff80001045c5a8: b.ne ffff80001045c63c // b.any : get_current(): 0.00 : ffff80001045c5ac: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff80001045c5b0: ldr w2, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff80001045c5b4: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001045c5b8: str w2, [x1, #16] : __percpu_add_case_64(): 0.00 : ffff80001045c5bc: mov x3, #0x1 // #1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001045c5c0: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045c5c4: add x0, x0, x2 0.00 : ffff80001045c5c8: ldxr x5, [x0] 0.00 : ffff80001045c5cc: add x5, x5, x3 0.00 : ffff80001045c5d0: stxr w4, x5, [x0] 0.00 : ffff80001045c5d4: cbnz w4, ffff80001045c5c8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045c5d8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045c5dc: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001045c5e0: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045c5e4: cbz x0, ffff80001045c5f0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045c5e8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045c5ec: cbnz x0, ffff80001045c554 : percpu_ref_get_many(): : this_cpu_add(*percpu_count, nr); 0.00 : ffff80001045c5f0: bl ffff800010cd2e78 0.00 : ffff80001045c5f4: b ffff80001045c554 : __read_once_size(): 0.00 : ffff80001045c5f8: ldr x4, [x20, #848] : atomic64_fetch_add_unless(): : atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) : { : s64 c = atomic64_read(v); : : do { : if (unlikely(c == u)) 0.00 : ffff80001045c5fc: cbz x4, ffff80001045c590 : break; : } while (!atomic64_try_cmpxchg(v, &c, c + a)); 0.00 : ffff80001045c600: add x2, x4, #0x1 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001045c604: b ffff80001045c634 0.00 : ffff80001045c608: b ffff80001045c634 : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff80001045c60c: mov x0, x3 0.00 : ffff80001045c610: mov x1, x4 0.00 : ffff80001045c614: mov x5, x1 0.00 : ffff80001045c618: casal x5, x2, [x3] 0.00 : ffff80001045c61c: mov x0, x5 : atomic64_try_cmpxchg(): : if (unlikely(r != o)) 0.00 : ffff80001045c620: cmp x0, x4 0.00 : ffff80001045c624: b.eq ffff80001045c554 // b.none 0.00 : ffff80001045c628: mov x4, x0 : atomic64_fetch_add_unless(): : if (unlikely(c == u)) 0.00 : ffff80001045c62c: cbnz x0, ffff80001045c600 0.00 : ffff80001045c630: b ffff80001045c590 : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff80001045c634: b ffff80001045ceec 0.00 : ffff80001045c638: b ffff80001045c620 : percpu_ref_get_many(): : atomic_long_add(nr, &ref->count); 0.00 : ffff80001045c63c: add x1, x21, #0x398 : arch_static_branch_jump(): 0.00 : ffff80001045c640: b ffff80001045c654 0.00 : ffff80001045c644: b ffff80001045c654 : __lse_atomic64_add(): : ATOMIC64_OP(add, stadd) 0.00 : ffff80001045c648: mov x0, #0x1 // #1 0.00 : ffff80001045c64c: stadd x0, [x1] 0.00 : ffff80001045c650: b ffff80001045c554 : __ll_sc_atomic64_add(): : ATOMIC64_OPS(add, add, I) 0.00 : ffff80001045c654: add x2, x21, #0x398 0.00 : ffff80001045c658: b ffff80001045cf0c 0.00 : ffff80001045c65c: b ffff80001045c554 Percent | Source code & Disassembly of vmlinux for cycles (86 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc2748 : : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 31.62 : ffff800010cc2748: ldr x4, [x0, #8] : plist_del(): : */ : void plist_del(struct plist_node *node, struct plist_head *head) : { : plist_check_head(head); : : if (!list_empty(&node->prio_list)) { 0.00 : ffff800010cc274c: add x3, x0, #0x8 20.90 : ffff800010cc2750: ldr x2, [x0, #24] 0.00 : ffff800010cc2754: cmp x3, x4 0.00 : ffff800010cc2758: b.eq ffff800010cc2790 // b.none : if (node->node_list.next != &head->node_list) { 0.00 : ffff800010cc275c: cmp x1, x2 0.00 : ffff800010cc2760: ldr x4, [x0, #8] 0.00 : ffff800010cc2764: b.eq ffff800010cc2778 // b.none : __read_once_size(): 0.00 : ffff800010cc2768: ldur x5, [x2, #-16] : plist_del(): : : next = list_entry(node->node_list.next, : struct plist_node, node_list); : : /* add the next plist_node into prio_list */ : if (list_empty(&next->prio_list)) 0.00 : ffff800010cc276c: sub x1, x2, #0x10 0.00 : ffff800010cc2770: cmp x1, x5 0.00 : ffff800010cc2774: b.eq ffff800010cc27ac // b.none : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff800010cc2778: ldr x1, [x0, #16] : __list_del(): : next->prev = prev; 0.00 : ffff800010cc277c: str x1, [x4, #8] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010cc2780: str x4, [x1] 0.00 : ffff800010cc2784: str x3, [x0, #8] 0.00 : ffff800010cc2788: ldr x2, [x0, #24] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010cc278c: str x3, [x0, #16] : __list_del_entry(): : __list_del(entry->prev, entry->next); 5.85 : ffff800010cc2790: ldr x3, [x0, #32] : plist_del(): : list_add(&next->prio_list, &node->prio_list); : } : list_del_init(&node->prio_list); : } : : list_del_init(&node->node_list); 0.00 : ffff800010cc2794: add x1, x0, #0x18 : __list_del(): : next->prev = prev; 4.85 : ffff800010cc2798: str x3, [x2, #8] : __write_once_size(): 30.90 : ffff800010cc279c: str x2, [x3] 5.88 : ffff800010cc27a0: str x1, [x0, #24] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010cc27a4: str x1, [x0, #32] : plist_del(): : : plist_check_head(head); : } 0.00 : ffff800010cc27a8: ret : __list_add(): : next->prev = new; 0.00 : ffff800010cc27ac: str x1, [x4, #8] : new->prev = prev; 0.00 : ffff800010cc27b0: stp x4, x3, [x2, #-16] : __write_once_size(): 0.00 : ffff800010cc27b4: str x1, [x0, #8] 0.00 : ffff800010cc27b8: ldr x4, [x0, #8] 0.00 : ffff800010cc27bc: b ffff800010cc2778 Percent | Source code & Disassembly of vmlinux for cycles (63 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101150a8 : : deactivate_task(): : : p->on_rq = TASK_ON_RQ_QUEUED; : } : : void deactivate_task(struct rq *rq, struct task_struct *p, int flags) : { 3.23 : ffff8000101150a8: stp x29, x30, [sp, #-48]! : p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; 0.00 : ffff8000101150ac: tst x2, #0x1 : { 0.00 : ffff8000101150b0: mov x29, sp 11.55 : ffff8000101150b4: str x19, [sp, #16] 0.00 : ffff8000101150b8: mov x19, x0 : p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; 0.00 : ffff8000101150bc: cset w0, eq // eq = none : : if (task_contributes_to_load(p)) 3.03 : ffff8000101150c0: ldr x3, [x1, #24] : p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; 0.00 : ffff8000101150c4: lsl w0, w0, #1 16.04 : ffff8000101150c8: str w0, [x1, #104] : if (task_contributes_to_load(p)) 0.00 : ffff8000101150cc: tbz w3, #1, ffff8000101150f0 0.00 : ffff8000101150d0: ldr w0, [x1, #44] 0.00 : ffff8000101150d4: tbnz w0, #16, ffff8000101150f0 0.00 : ffff8000101150d8: ldr x0, [x1, #24] 0.00 : ffff8000101150dc: tbnz w0, #10, ffff8000101150f0 : rq->nr_uninterruptible++; 0.00 : ffff8000101150e0: ldr x0, [x19, #2344] 0.00 : ffff8000101150e4: add x0, x0, #0x1 0.00 : ffff8000101150e8: str x0, [x19, #2344] 0.00 : ffff8000101150ec: nop : dequeue_task(): : if (!(flags & DEQUEUE_NOCLOCK)) 1.63 : ffff8000101150f0: tbnz w2, #3, ffff8000101150fc : update_rq_clock(): : if (rq->clock_update_flags & RQCF_ACT_SKIP) 0.00 : ffff8000101150f4: ldr w0, [x19, #2392] 0.00 : ffff8000101150f8: tbz w0, #1, ffff800010115154 : dequeue_task(): : if (!(flags & DEQUEUE_SAVE)) { 0.00 : ffff8000101150fc: tbnz w2, #1, ffff800010115120 : sched_info_on(): : { : #ifdef CONFIG_SCHEDSTATS : return 1; : #elif defined(CONFIG_TASK_DELAY_ACCT) : extern int delayacct_on; : return delayacct_on; 1.49 : ffff800010115100: adrp x3, ffff80001190a000 <__per_cpu_offset+0x6d8> : sched_info_dequeued(): : * from dequeue_task() to account for possible rq->clock skew across CPUs. The : * delta taken on each CPU would annul the skew. : */ : static inline void sched_info_dequeued(struct rq *rq, struct task_struct *t) : { : unsigned long long now = rq_clock(rq), delta = 0; 0.00 : ffff800010115104: mov x0, #0x0 // #0 : : if (sched_info_on()) { 4.62 : ffff800010115108: ldr w3, [x3, #792] 0.00 : ffff80001011510c: cbnz w3, ffff80001011513c : if (t->sched_info.last_queued) : delta = now - t->sched_info.last_queued; : } : sched_info_reset_dequeued(t); : t->sched_info.run_delay += delta; 3.18 : ffff800010115110: ldr x3, [x1, #848] : sched_info_reset_dequeued(): : t->sched_info.last_queued = 0; 7.79 : ffff800010115114: str xzr, [x1, #864] : sched_info_dequeued(): : t->sched_info.run_delay += delta; 0.00 : ffff800010115118: add x0, x3, x0 0.00 : ffff80001011511c: str x0, [x1, #848] : dequeue_task(): : p->sched_class->dequeue_task(rq, p, flags); 6.08 : ffff800010115120: ldr x3, [x1, #128] 0.00 : ffff800010115124: mov x0, x19 16.04 : ffff800010115128: ldr x3, [x3, #16] 0.00 : ffff80001011512c: blr x3 : deactivate_task(): : : dequeue_task(rq, p, flags); : } 1.50 : ffff800010115130: ldr x19, [sp, #16] 3.15 : ffff800010115134: ldp x29, x30, [sp], #48 0.00 : ffff800010115138: ret : sched_info_dequeued(): : if (t->sched_info.last_queued) 4.72 : ffff80001011513c: ldr x0, [x1, #864] : unsigned long long now = rq_clock(rq), delta = 0; 7.86 : ffff800010115140: ldr x3, [x19, #2400] : delta = now - t->sched_info.last_queued; 0.00 : ffff800010115144: cmp x0, #0x0 0.00 : ffff800010115148: sub x3, x3, x0 0.00 : ffff80001011514c: csel x0, x3, x0, ne // ne = any 8.09 : ffff800010115150: b ffff800010115110 0.00 : ffff800010115154: str w2, [x29, #36] : update_rq_clock(): 0.00 : ffff800010115158: mov x0, x19 0.00 : ffff80001011515c: str x1, [x29, #40] 0.00 : ffff800010115160: bl ffff800010113d48 0.00 : ffff800010115164: ldr w2, [x29, #36] 0.00 : ffff800010115168: ldr x1, [x29, #40] 0.00 : ffff80001011516c: b ffff8000101150fc Percent | Source code & Disassembly of vmlinux for cycles (62 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001010bc70 : : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001010bc70: mrs x0, sp_el0 : to_kthread(): : current->set_child_tid = (__force void __user *)kthread; : } : : static inline struct kthread *to_kthread(struct task_struct *k) : { : WARN_ON(!(k->flags & PF_KTHREAD)); 0.00 : ffff80001010bc74: ldr w1, [x0, #44] 0.00 : ffff80001010bc78: tbz w1, #21, ffff80001010bc8c : return (__force void *)k->set_child_tid; 35.05 : ffff80001010bc7c: ldr x0, [x0, #1344] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 64.95 : ffff80001010bc80: ldr x0, [x0] : kthread_should_stop(): : * and this will return true. You should then return, and your return : * value will be passed through to kthread_stop(). : */ : bool kthread_should_stop(void) : { : return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags); 0.00 : ffff80001010bc84: ubfx w0, w0, #1, #1 : } 0.00 : ffff80001010bc88: ret : to_kthread(): : WARN_ON(!(k->flags & PF_KTHREAD)); 0.00 : ffff80001010bc8c: brk #0x800 : return (__force void *)k->set_child_tid; 0.00 : ffff80001010bc90: ldr x0, [x0, #1344] : test_bit(): 0.00 : ffff80001010bc94: ldr x0, [x0] : kthread_should_stop(): : return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags); 0.00 : ffff80001010bc98: ubfx w0, w0, #1, #1 : } 0.00 : ffff80001010bc9c: ret Percent | Source code & Disassembly of vmlinux for cycles (99 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010085588 : : fpsimd_save(): : /* : * Ensure FPSIMD/SVE storage in memory for the loaded context is up to : * date with respect to the CPU registers. : */ : static void fpsimd_save(void) : { 0.00 : ffff800010085588: stp x29, x30, [sp, #-48]! 0.00 : ffff80001008558c: mov x29, sp 2.02 : ffff800010085590: stp x19, x20, [sp, #16] : struct fpsimd_last_state_struct const *last = : this_cpu_ptr(&fpsimd_last_state); 0.00 : ffff800010085594: adrp x19, ffff80001151d000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 5.22 : ffff800010085598: mrs x20, tpidr_el1 : fpsimd_save(): 10.00 : ffff80001008559c: add x19, x19, #0x60 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000100855a0: b ffff8000100855ec : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff8000100855a4: nop : fpsimd_save(): : /* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */ : : WARN_ON(!system_supports_fpsimd()); : WARN_ON(!have_cpu_fpsimd_context()); 0.00 : ffff8000100855a8: bl ffff800010085440 32.19 : ffff8000100855ac: tst w0, #0xff 0.00 : ffff8000100855b0: b.eq ffff800010085610 // b.none : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000100855b4: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 12.27 : ffff8000100855b8: ldr x0, [x0] : fpsimd_save(): : : if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { 0.00 : ffff8000100855bc: tst w0, #0x8 0.00 : ffff8000100855c0: b.ne ffff8000100855e0 // b.any 0.00 : ffff8000100855c4: str x21, [x29, #32] : this_cpu_ptr(&fpsimd_last_state); 0.00 : ffff8000100855c8: add x21, x19, x20 : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff8000100855cc: b ffff800010085618 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff8000100855d0: nop : fpsimd_save(): : : sve_save_state((char *)last->sve_state + : sve_ffr_offset(last->sve_vl), : &last->st->fpsr); : } else : fpsimd_save_state(last->st); 0.00 : ffff8000100855d4: ldr x0, [x19, x20] 0.00 : ffff8000100855d8: bl ffff80001008725c 1.10 : ffff8000100855dc: ldr x21, [x29, #32] : } : } 8.80 : ffff8000100855e0: ldp x19, x20, [sp, #16] 28.42 : ffff8000100855e4: ldp x29, x30, [sp], #48 0.00 : ffff8000100855e8: ret : test_bit(): 0.00 : ffff8000100855ec: adrp x0, ffff800011af2000 0.00 : ffff8000100855f0: ldr x0, [x0, #1320] : fpsimd_save(): : WARN_ON(!system_supports_fpsimd()); 0.00 : ffff8000100855f4: tst w0, #0x10000 0.00 : ffff8000100855f8: b.eq ffff8000100855a8 // b.none 0.00 : ffff8000100855fc: brk #0x800 : WARN_ON(!have_cpu_fpsimd_context()); 0.00 : ffff800010085600: bl ffff800010085440 0.00 : ffff800010085604: tst w0, #0xff 0.00 : ffff800010085608: b.ne ffff8000100855b4 // b.any 0.00 : ffff80001008560c: nop 0.00 : ffff800010085610: brk #0x800 0.00 : ffff800010085614: b ffff8000100855b4 : test_bit(): 0.00 : ffff800010085618: adrp x0, ffff800011af2000 0.00 : ffff80001008561c: ldr x0, [x0, #1320] : fpsimd_save(): : if (system_supports_sve() && test_thread_flag(TIF_SVE)) { 0.00 : ffff800010085620: tst w0, #0x400000 0.00 : ffff800010085624: b.eq ffff8000100855d4 // b.none : get_current(): 0.00 : ffff800010085628: mrs x0, sp_el0 : test_bit(): 0.00 : ffff80001008562c: ldr x0, [x0] : fpsimd_save(): 0.00 : ffff800010085630: tst w0, #0x800000 0.00 : ffff800010085634: b.eq ffff8000100855d4 // b.none : if (WARN_ON(sve_get_vl() != last->sve_vl)) { 0.00 : ffff800010085638: bl ffff8000100874e4 0.00 : ffff80001008563c: ldr w1, [x21, #16] 0.00 : ffff800010085640: cmp w0, w1 0.00 : ffff800010085644: b.ne ffff800010085684 // b.any : sve_ffr_offset(): : #define SVE_VL_ARCH_MAX 0x100 : : /* Offset of FFR in the SVE register dump */ : static inline size_t sve_ffr_offset(int vl) : { : return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET; 0.00 : ffff800010085648: cmp w0, #0x0 0.00 : ffff80001008564c: add w2, w0, #0xf 0.00 : ffff800010085650: csel w2, w2, w0, lt // lt = tstop : fpsimd_save(): : &last->st->fpsr); 0.00 : ffff800010085654: ldr x1, [x19, x20] : sve_save_state((char *)last->sve_state + 0.00 : ffff800010085658: ldr x0, [x21, #8] : sve_ffr_offset(): 0.00 : ffff80001008565c: asr w2, w2, #4 : fpsimd_save(): 0.00 : ffff800010085660: add x1, x1, #0x200 : sve_ffr_offset(): 0.00 : ffff800010085664: add w2, w2, w2, lsl #4 0.00 : ffff800010085668: lsl w2, w2, #5 : fpsimd_save(): 0.00 : ffff80001008566c: add x0, x0, x2 0.00 : ffff800010085670: bl ffff800010087310 0.00 : ffff800010085674: ldr x21, [x29, #32] : } 0.00 : ffff800010085678: ldp x19, x20, [sp, #16] 0.00 : ffff80001008567c: ldp x29, x30, [sp], #48 0.00 : ffff800010085680: ret : if (WARN_ON(sve_get_vl() != last->sve_vl)) { 0.00 : ffff800010085684: brk #0x800 : force_signal_inject(SIGKILL, SI_KERNEL, 0); 0.00 : ffff800010085688: mov x2, #0x0 // #0 0.00 : ffff80001008568c: mov w1, #0x80 // #128 0.00 : ffff800010085690: mov w0, #0x9 // #9 0.00 : ffff800010085694: bl ffff80001008ede0 : return; 0.00 : ffff800010085698: ldr x21, [x29, #32] 0.00 : ffff80001008569c: b ffff8000100855e0 Percent | Source code & Disassembly of vmlinux for cycles (61 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016b9f8 : : hrtimer_try_to_cancel(): : * * 1 when the timer was active : * * -1 when the timer is currently executing the callback function and : * cannot be stopped : */ : int hrtimer_try_to_cancel(struct hrtimer *timer) : { 0.00 : ffff80001016b9f8: stp x29, x30, [sp, #-64]! 0.00 : ffff80001016b9fc: mov x29, sp 0.00 : ffff80001016ba00: stp x19, x20, [sp, #16] 0.00 : ffff80001016ba04: adrp x19, ffff800011909000 1.68 : ffff80001016ba08: str x21, [sp, #32] 0.00 : ffff80001016ba0c: add x1, x19, #0x908 0.00 : ffff80001016ba10: mov x21, x0 0.00 : ffff80001016ba14: mov w20, #0x0 // #0 9.57 : ffff80001016ba18: ldr x2, [x1] 0.00 : ffff80001016ba1c: str x2, [x29, #56] 0.00 : ffff80001016ba20: mov x2, #0x0 // #0 : * Check lockless first. If the timer is not active (neither : * enqueued nor running the callback, nothing to do here. The : * base lock does not serialize against a concurrent enqueue, : * so we can avoid taking it. : */ : if (!hrtimer_active(timer)) 0.00 : ffff80001016ba24: bl ffff80001016aeb8 0.00 : ffff80001016ba28: tst w0, #0xff 0.00 : ffff80001016ba2c: b.ne ffff80001016ba58 // b.any : : unlock_hrtimer_base(timer, &flags); : : return ret; : : } 0.00 : ffff80001016ba30: add x19, x19, #0x908 0.00 : ffff80001016ba34: mov w0, w20 0.00 : ffff80001016ba38: ldr x2, [x29, #56] 0.00 : ffff80001016ba3c: ldr x1, [x19] 0.00 : ffff80001016ba40: eor x1, x2, x1 0.00 : ffff80001016ba44: cbnz x1, ffff80001016bac8 0.00 : ffff80001016ba48: ldp x19, x20, [sp, #16] 0.00 : ffff80001016ba4c: ldr x21, [sp, #32] 0.00 : ffff80001016ba50: ldp x29, x30, [sp], #64 0.00 : ffff80001016ba54: ret : base = lock_hrtimer_base(timer, &flags); 0.00 : ffff80001016ba58: add x1, x29, #0x30 0.00 : ffff80001016ba5c: mov x0, x21 6.68 : ffff80001016ba60: bl ffff80001016adc8 : int ret = -1; 3.32 : ffff80001016ba64: mov w20, #0xffffffff // #-1 : hrtimer_callback_running(): : * Helper function to check, whether the timer is running the callback : * function : */ : static inline int hrtimer_callback_running(struct hrtimer *timer) : { : return timer->base->running == timer; 1.67 : ffff80001016ba68: ldr x2, [x21, #48] : hrtimer_try_to_cancel(): : if (!hrtimer_callback_running(timer)) 0.00 : ffff80001016ba6c: ldr x3, [x2, #24] 0.00 : ffff80001016ba70: cmp x21, x3 0.00 : ffff80001016ba74: b.eq ffff80001016bab8 // b.none : remove_hrtimer(): : if (state & HRTIMER_STATE_ENQUEUED) { 0.00 : ffff80001016ba78: ldrb w1, [x21, #56] : return 0; 0.00 : ffff80001016ba7c: mov w20, #0x0 // #0 : if (state & HRTIMER_STATE_ENQUEUED) { 1.70 : ffff80001016ba80: tbz w1, #0, ffff80001016bab8 : reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); 3.36 : ffff80001016ba84: ldr x4, [x0] : __remove_hrtimer(timer, base, state, reprogram); 0.00 : ffff80001016ba88: mov x1, x0 : reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016ba8c: adrp x3, ffff800011525000 0.00 : ffff80001016ba90: add x3, x3, #0x980 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001016ba94: mrs x0, tpidr_el1 : remove_hrtimer(): 0.00 : ffff80001016ba98: add x3, x3, x0 : __remove_hrtimer(timer, base, state, reprogram); 0.00 : ffff80001016ba9c: mov w2, #0x0 // #0 : reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016baa0: cmp x4, x3 : __remove_hrtimer(timer, base, state, reprogram); 0.00 : ffff80001016baa4: mov x0, x21 1.69 : ffff80001016baa8: cset w3, eq // eq = none : return 1; 0.00 : ffff80001016baac: mov w20, #0x1 // #1 : __remove_hrtimer(timer, base, state, reprogram); 0.00 : ffff80001016bab0: bl ffff80001016b130 <__remove_hrtimer> 0.00 : ffff80001016bab4: ldr x2, [x21, #48] : unlock_hrtimer_base(): : raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); 27.69 : ffff80001016bab8: ldr x0, [x2] 42.63 : ffff80001016babc: ldr x1, [x29, #48] 0.00 : ffff80001016bac0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff80001016bac4: b ffff80001016ba30 : hrtimer_try_to_cancel(): : } 0.00 : ffff80001016bac8: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (61 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b8b8 : : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001017b8b8: add w3, w0, #0x3f 0.00 : ffff80001017b8bc: cmp w0, #0x0 0.00 : ffff80001017b8c0: csel w3, w3, w0, lt // lt = tstop : can_stop_idle_tick(): : else if (ts->tick_stopped) : tick_nohz_restart_sched_tick(ts, ktime_get()); : #endif : } : : static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) 0.00 : ffff80001017b8c4: stp x29, x30, [sp, #-32]! : test_bit(): 0.00 : ffff80001017b8c8: adrp x4, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001017b8cc: add x4, x4, #0x160 0.00 : ffff80001017b8d0: asr w3, w3, #6 : can_stop_idle_tick(): 0.00 : ffff80001017b8d4: mov x29, sp : test_bit(): 0.00 : ffff80001017b8d8: sxtw x3, w3 : can_stop_idle_tick(): 16.36 : ffff80001017b8dc: str x19, [sp, #16] : test_bit(): 1.69 : ffff80001017b8e0: ldr x3, [x4, x3, lsl #3] 0.00 : ffff80001017b8e4: lsr x3, x3, x0 : can_stop_idle_tick(): : * jiffies, then give up the assignment and let it be taken by : * the CPU which runs the tick timer next. If we don't drop : * this here the jiffies might be stale and do_timer() never : * invoked. : */ : if (unlikely(!cpu_online(cpu))) { 0.00 : ffff80001017b8e8: tbz w3, #0, ffff80001017b93c : */ : ts->next_tick = 0; : return false; : } : : if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) 44.33 : ffff80001017b8ec: cbz w1, ffff80001017b928 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 14.71 : ffff80001017b8f0: mrs x0, sp_el0 : test_bit(): 0.00 : ffff80001017b8f4: ldr x0, [x0] : can_stop_idle_tick(): : return false; : : if (need_resched()) 0.00 : ffff80001017b8f8: tst w0, #0x2 0.00 : ffff80001017b8fc: b.ne ffff80001017b928 // b.any : return false; : : if (unlikely(local_softirq_pending())) { 0.00 : ffff80001017b900: adrp x0, ffff800011528000 0.00 : ffff80001017b904: add x0, x0, #0x7c0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001017b908: mrs x1, tpidr_el1 : can_stop_idle_tick(): 0.00 : ffff80001017b90c: ldr w1, [x0, x1] : /* Should not happen for nohz-full */ : if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) : return false; : } : : return true; 0.00 : ffff80001017b910: mov w19, #0x1 // #1 : if (unlikely(local_softirq_pending())) { 0.00 : ffff80001017b914: cbnz w1, ffff80001017b96c : } 4.92 : ffff80001017b918: mov w0, w19 6.76 : ffff80001017b91c: ldr x19, [sp, #16] 11.23 : ffff80001017b920: ldp x29, x30, [sp], #32 0.00 : ffff80001017b924: ret : return false; 0.00 : ffff80001017b928: mov w19, #0x0 // #0 : } 0.00 : ffff80001017b92c: mov w0, w19 0.00 : ffff80001017b930: ldr x19, [sp, #16] 0.00 : ffff80001017b934: ldp x29, x30, [sp], #32 0.00 : ffff80001017b938: ret : if (cpu == tick_do_timer_cpu) 0.00 : ffff80001017b93c: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001017b940: ldr w3, [x1, #712] 0.00 : ffff80001017b944: cmp w0, w3 0.00 : ffff80001017b948: b.ne ffff80001017b954 // b.any : tick_do_timer_cpu = TICK_DO_TIMER_NONE; 0.00 : ffff80001017b94c: mov w0, #0xffffffff // #-1 0.00 : ffff80001017b950: str w0, [x1, #712] : return false; 0.00 : ffff80001017b954: mov w19, #0x0 // #0 : ts->next_tick = 0; 0.00 : ffff80001017b958: str xzr, [x2] : } 0.00 : ffff80001017b95c: mov w0, w19 0.00 : ffff80001017b960: ldr x19, [sp, #16] 0.00 : ffff80001017b964: ldp x29, x30, [sp], #32 0.00 : ffff80001017b968: ret 0.00 : ffff80001017b96c: str x20, [x29, #24] : if (ratelimit < 10 && 0.00 : ffff80001017b970: adrp x20, ffff800011b22000 0.00 : ffff80001017b974: add x20, x20, #0x530 : return false; 0.00 : ffff80001017b978: mov w19, #0x0 // #0 : if (ratelimit < 10 && 0.00 : ffff80001017b97c: ldr w0, [x20, #12] 0.00 : ffff80001017b980: cmp w0, #0x9 0.00 : ffff80001017b984: b.gt ffff80001017b9b0 0.00 : ffff80001017b988: tst w1, #0xfffffdff 0.00 : ffff80001017b98c: b.eq ffff80001017b9b0 // b.none : pr_warn("NOHZ: local_softirq_pending %02x\n", 0.00 : ffff80001017b990: adrp x0, ffff8000111c9000 0.00 : ffff80001017b994: add x0, x0, #0x808 0.00 : ffff80001017b998: bl ffff80001014a85c : ratelimit++; 0.00 : ffff80001017b99c: ldr w0, [x20, #12] 0.00 : ffff80001017b9a0: add w0, w0, #0x1 0.00 : ffff80001017b9a4: str w0, [x20, #12] 0.00 : ffff80001017b9a8: ldr x20, [x29, #24] 0.00 : ffff80001017b9ac: b ffff80001017b918 0.00 : ffff80001017b9b0: ldr x20, [x29, #24] 0.00 : ffff80001017b9b4: b ffff80001017b918 Percent | Source code & Disassembly of vmlinux for cycles (60 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016adc8 : : lock_hrtimer_base(): : * remains locked. : */ : static : struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, : unsigned long *flags) : { 13.25 : ffff80001016adc8: stp x29, x30, [sp, #-48]! 0.00 : ffff80001016adcc: mov x29, sp 13.20 : ffff80001016add0: stp x19, x20, [sp, #16] 0.00 : ffff80001016add4: adrp x19, ffff800011925000 : struct hrtimer_clock_base *base; : : for (;;) { : base = READ_ONCE(timer->base); : if (likely(base != &migration_base)) { 0.00 : ffff80001016add8: add x19, x19, #0x180 : { 4.80 : ffff80001016addc: stp x21, x22, [sp, #32] : if (likely(base != &migration_base)) { 0.00 : ffff80001016ade0: add x19, x19, #0x40 : { 0.00 : ffff80001016ade4: mov x21, x0 0.00 : ffff80001016ade8: mov x22, x1 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 15.30 : ffff80001016adec: ldr x20, [x21, #48] : lock_hrtimer_base(): : if (likely(base != &migration_base)) { 0.00 : ffff80001016adf0: cmp x20, x19 0.00 : ffff80001016adf4: b.eq ffff80001016ae2c // b.none : raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); 25.00 : ffff80001016adf8: ldr x0, [x20] 0.00 : ffff80001016adfc: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff80001016ae00: str x0, [x22] 0.00 : ffff80001016ae04: mov x1, x0 : if (likely(base == timer->base)) 21.79 : ffff80001016ae08: ldr x0, [x21, #48] 0.00 : ffff80001016ae0c: cmp x0, x20 0.00 : ffff80001016ae10: b.ne ffff80001016ae24 // b.any : /* The timer has migrated to another CPU: */ : raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); : } : cpu_relax(); : } : } 6.67 : ffff80001016ae14: ldp x19, x20, [sp, #16] 0.00 : ffff80001016ae18: ldp x21, x22, [sp, #32] 0.00 : ffff80001016ae1c: ldp x29, x30, [sp], #48 0.00 : ffff80001016ae20: ret : raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); 0.00 : ffff80001016ae24: ldr x0, [x20] 0.00 : ffff80001016ae28: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff80001016ae2c: yield 0.00 : ffff80001016ae30: b ffff80001016adec Percent | Source code & Disassembly of vmlinux for cycles (56 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010251f98 <__slab_alloc.isra.97>: : __slab_alloc(): : : /* : * Another one that disabled interrupt and compensates for possible : * cpu changes by refetching the per cpu area pointer. : */ : static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 10.60 : ffff800010251f98: stp x29, x30, [sp, #-32]! 0.00 : ffff800010251f9c: mov x29, sp 3.49 : ffff800010251fa0: str x19, [sp, #16] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 1.75 : ffff800010251fa4: mrs x19, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010251fa8: and w4, w19, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff800010251fac: cbnz w4, ffff800010251fb8 <__slab_alloc.isra.97+0x20> : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010251fb0: mov x4, #0x60 // #96 0.00 : ffff800010251fb4: msr daifset, #0x2 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010251fb8: mrs x5, tpidr_el1 : __slab_alloc(): : /* : * We may have been preempted and rescheduled on a different : * cpu before disabling interrupts. Need to reload cpu area : * pointer. : */ : c = this_cpu_ptr(s->cpu_slab); 0.00 : ffff800010251fbc: ldr x4, [x0] : #endif : : p = ___slab_alloc(s, gfpflags, node, addr, c); 0.00 : ffff800010251fc0: add x4, x4, x5 0.00 : ffff800010251fc4: bl ffff800010251990 <___slab_alloc> : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff800010251fc8: msr daif, x19 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 10.91 : ffff800010251fcc: nop : __slab_alloc(): : local_irq_restore(flags); : return p; : } 0.00 : ffff800010251fd0: ldr x19, [sp, #16] 0.00 : ffff800010251fd4: ldp x29, x30, [sp], #32 0.00 : ffff800010251fd8: ret : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 1.74 : ffff800010251fdc: dsb sy : __slab_alloc(): 60.79 : ffff800010251fe0: ldr x19, [sp, #16] 10.72 : ffff800010251fe4: ldp x29, x30, [sp], #32 0.00 : ffff800010251fe8: ret Percent | Source code & Disassembly of vmlinux for cycles (155 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010718568 : : alloc_iova_fast(): : * fails too and the flush_rcache flag is set then the rcache will be flushed. : */ : unsigned long : alloc_iova_fast(struct iova_domain *iovad, unsigned long size, : unsigned long limit_pfn, bool flush_rcache) : { 1.90 : ffff800010718568: stp x29, x30, [sp, #-96]! : __order_base_2(): : ) : : static inline __attribute_const__ : int __order_base_2(unsigned long n) : { : return n > 1 ? ilog2(n - 1) + 1 : 0; 0.00 : ffff80001071856c: cmp x1, #0x1 : alloc_iova_fast(): 0.00 : ffff800010718570: mov x29, sp 11.72 : ffff800010718574: stp x19, x20, [sp, #16] 0.00 : ffff800010718578: and w20, w3, #0xff 0.00 : ffff80001071857c: stp x21, x22, [sp, #32] 1.25 : ffff800010718580: mov x21, x0 0.00 : ffff800010718584: stp x23, x24, [sp, #48] 0.00 : ffff800010718588: mov x24, x1 6.38 : ffff80001071858c: str x25, [sp, #64] 0.00 : ffff800010718590: mov x25, x2 : __order_base_2(): 0.00 : ffff800010718594: b.ls ffff800010718614 // b.plast 0.00 : ffff800010718598: sub x0, x1, #0x1 : __fls(): : * : * Undefined if no set bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __fls(unsigned long word) : { : return (sizeof(word) * 8) - 1 - __builtin_clzl(word); 0.00 : ffff80001071859c: mov x19, #0x3f // #63 0.00 : ffff8000107185a0: clz x0, x0 0.00 : ffff8000107185a4: sub x19, x19, x0 : fls64(): : #elif BITS_PER_LONG == 64 : static __always_inline int fls64(__u64 x) : { : if (x == 0) : return 0; : return __fls(x) + 1; 0.00 : ffff8000107185a8: add w19, w19, #0x1 : iova_rcache_get(): : unsigned long size, : unsigned long limit_pfn) : { : unsigned int log_size = order_base_2(size); : : if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE) 0.00 : ffff8000107185ac: cmp w19, #0x5 0.00 : ffff8000107185b0: b.ls ffff8000107187dc // b.plast 0.00 : ffff8000107185b4: adrp x23, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000107185b8: adrp x22, ffff80001190a000 <__per_cpu_offset+0x6d8> : alloc_iova_fast(): : for_each_online_cpu(cpu) 0.00 : ffff8000107185bc: add x23, x23, #0x160 0.00 : ffff8000107185c0: add x22, x22, #0x2f0 0.00 : ffff8000107185c4: nop : new_iova = alloc_iova(iovad, size, limit_pfn, true); 0.00 : ffff8000107185c8: mov w3, #0x1 // #1 0.00 : ffff8000107185cc: mov x2, x25 0.00 : ffff8000107185d0: mov x1, x24 0.00 : ffff8000107185d4: mov x0, x21 0.00 : ffff8000107185d8: bl ffff800010717540 : if (!new_iova) { 0.00 : ffff8000107185dc: cbnz x0, ffff80001071874c : if (!flush_rcache) 0.00 : ffff8000107185e0: cbz w20, ffff80001071876c : for_each_online_cpu(cpu) 0.00 : ffff8000107185e4: mov w19, #0xffffffff // #-1 0.00 : ffff8000107185e8: mov x1, x23 0.00 : ffff8000107185ec: mov w0, w19 0.00 : ffff8000107185f0: bl ffff800010cb92d8 : flush_rcache = false; 0.00 : ffff8000107185f4: mov w20, #0x0 // #0 : for_each_online_cpu(cpu) 0.00 : ffff8000107185f8: ldr w1, [x22] 0.00 : ffff8000107185fc: mov w19, w0 0.00 : ffff800010718600: cmp w0, w1 0.00 : ffff800010718604: b.cs ffff8000107185c8 // b.hs, b.nlast : free_cpu_cached_iovas(cpu, iovad); 0.00 : ffff800010718608: mov x1, x21 0.00 : ffff80001071860c: bl ffff8000107184d8 0.00 : ffff800010718610: b ffff8000107185e8 1.33 : ffff800010718614: stp x26, x27, [x29, #72] : __order_base_2(): 0.00 : ffff800010718618: mov x23, #0x80 // #128 : iova_rcache_get(): : unsigned int log_size = order_base_2(size); 0.00 : ffff80001071861c: mov x19, #0x0 // #0 : __iova_rcache_get(): : cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches); 0.66 : ffff800010718620: add x0, x19, x19, lsl #3 : iova_rcache_get(): : return 0; : : return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size); 1.31 : ffff800010718624: sub x26, x25, x24 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010718628: mrs x1, tpidr_el1 : __iova_rcache_get(): : cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches); 0.00 : ffff80001071862c: lsl x0, x0, #2 : iova_rcache_get(): : return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size); 0.00 : ffff800010718630: add x26, x26, #0x1 : __iova_rcache_get(): : cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches); 0.00 : ffff800010718634: sub x0, x0, x19 0.00 : ffff800010718638: add x0, x21, x0, lsl #3 0.00 : ffff80001071863c: ldr x22, [x0, #400] 9.80 : ffff800010718640: add x22, x22, x1 : spin_lock_irqsave(&cpu_rcache->lock, flags); 0.00 : ffff800010718644: mov x0, x22 0.00 : ffff800010718648: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff80001071864c: mov x27, x0 : if (!iova_magazine_empty(cpu_rcache->loaded)) { 23.05 : ffff800010718650: ldr x3, [x22, #8] : iova_magazine_empty(): : return (!mag || mag->size == 0); 0.00 : ffff800010718654: cbz x3, ffff800010718660 16.27 : ffff800010718658: ldr x5, [x3] 0.00 : ffff80001071865c: cbnz x5, ffff8000107186d0 0.00 : ffff800010718660: str x28, [x29, #88] : __iova_rcache_get(): : } else if (!iova_magazine_empty(cpu_rcache->prev)) { 0.00 : ffff800010718664: ldr x0, [x22, #16] : iova_magazine_empty(): : return (!mag || mag->size == 0); 0.00 : ffff800010718668: cbz x0, ffff800010718674 0.00 : ffff80001071866c: ldr x1, [x0] 0.00 : ffff800010718670: cbnz x1, ffff8000107186b8 : iova_rcache_get(): : return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size); 0.00 : ffff800010718674: add x23, x21, x23 : spin_lock(): : raw_spin_lock_init(&(_lock)->rlock); \ : } while (0) : : static __always_inline void spin_lock(spinlock_t *lock) : { : raw_spin_lock(&lock->rlock); 0.00 : ffff800010718678: mov x0, x23 0.00 : ffff80001071867c: bl ffff800010cd80f0 <_raw_spin_lock> : __iova_rcache_get(): : if (rcache->depot_size > 0) { 0.00 : ffff800010718680: add x0, x19, x19, lsl #3 0.00 : ffff800010718684: lsl x0, x0, #2 0.00 : ffff800010718688: sub x19, x0, x19 0.00 : ffff80001071868c: add x28, x21, x19, lsl #3 0.00 : ffff800010718690: ldr x0, [x28, #136] 0.00 : ffff800010718694: cbnz x0, ffff8000107187a0 : spin_unlock(): : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ : } while (0) : : static __always_inline void spin_unlock(spinlock_t *lock) : { : raw_spin_unlock(&lock->rlock); 0.00 : ffff800010718698: mov x0, x23 0.00 : ffff80001071869c: bl ffff800010cd7c38 <_raw_spin_unlock> : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000107186a0: mov x1, x27 0.00 : ffff8000107186a4: mov x0, x22 0.00 : ffff8000107186a8: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff8000107186ac: ldr x28, [x29, #88] 0.00 : ffff8000107186b0: ldp x26, x27, [x29, #72] 0.00 : ffff8000107186b4: b ffff8000107185b4 : __iova_rcache_get(): : swap(cpu_rcache->prev, cpu_rcache->loaded); 0.00 : ffff8000107186b8: stp x0, x3, [x22, #8] 0.00 : ffff8000107186bc: ldr x5, [x0] : iova_magazine_empty(): : return (!mag || mag->size == 0); 0.00 : ffff8000107186c0: cbnz x5, ffff8000107186c8 : iova_magazine_pop(): : BUG_ON(iova_magazine_empty(mag)); 0.00 : ffff8000107186c4: brk #0x800 0.00 : ffff8000107186c8: ldr x28, [x29, #88] : iova_magazine_empty(): : return (!mag || mag->size == 0); 0.00 : ffff8000107186cc: mov x3, x0 : iova_magazine_pop(): : for (i = mag->size - 1; mag->pfns[i] > limit_pfn; i--) 3.26 : ffff8000107186d0: sub w0, w5, #0x1 0.00 : ffff8000107186d4: sxtw x2, w0 0.00 : ffff8000107186d8: add x1, x3, x2, lsl #3 2.56 : ffff8000107186dc: ldr x19, [x1, #8] 1.31 : ffff8000107186e0: cmp x26, x19 0.00 : ffff8000107186e4: b.cs ffff800010718704 // b.hs, b.nlast : if (i == 0) 0.00 : ffff8000107186e8: cbz w0, ffff80001071878c : for (i = mag->size - 1; mag->pfns[i] > limit_pfn; i--) 0.00 : ffff8000107186ec: sub w0, w0, #0x1 0.00 : ffff8000107186f0: sxtw x2, w0 0.00 : ffff8000107186f4: add x4, x3, x2, lsl #3 0.00 : ffff8000107186f8: ldr x19, [x4, #8] 0.00 : ffff8000107186fc: cmp x26, x19 0.00 : ffff800010718700: b.cc ffff8000107186e8 // b.lo, b.ul, b.last : mag->pfns[i] = mag->pfns[--mag->size]; 3.78 : ffff800010718704: sub x5, x5, #0x1 1.31 : ffff800010718708: str x5, [x3] 0.00 : ffff80001071870c: add x2, x3, x2, lsl #3 : spin_unlock_irqrestore(): 0.00 : ffff800010718710: mov x1, x27 : iova_magazine_pop(): 0.00 : ffff800010718714: add x5, x3, x5, lsl #3 : spin_unlock_irqrestore(): 0.00 : ffff800010718718: mov x0, x22 : iova_magazine_pop(): 0.00 : ffff80001071871c: ldr x3, [x5, #8] 1.25 : ffff800010718720: str x3, [x2, #8] : spin_unlock_irqrestore(): 0.00 : ffff800010718724: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 2.55 : ffff800010718728: ldp x26, x27, [x29, #72] : alloc_iova_fast(): : if (iova_pfn) 0.00 : ffff80001071872c: cbz x19, ffff8000107185b4 : } 10.33 : ffff800010718730: mov x0, x19 0.00 : ffff800010718734: ldr x25, [sp, #64] 0.00 : ffff800010718738: ldp x19, x20, [sp, #16] 0.00 : ffff80001071873c: ldp x21, x22, [sp, #32] 0.00 : ffff800010718740: ldp x23, x24, [sp, #48] 0.00 : ffff800010718744: ldp x29, x30, [sp], #96 0.00 : ffff800010718748: ret : return new_iova->pfn_lo; 0.00 : ffff80001071874c: ldr x19, [x0, #32] : } 0.00 : ffff800010718750: ldp x21, x22, [sp, #32] 0.00 : ffff800010718754: mov x0, x19 0.00 : ffff800010718758: ldp x19, x20, [sp, #16] 0.00 : ffff80001071875c: ldp x23, x24, [sp, #48] 0.00 : ffff800010718760: ldr x25, [sp, #64] 0.00 : ffff800010718764: ldp x29, x30, [sp], #96 0.00 : ffff800010718768: ret : return 0; 0.00 : ffff80001071876c: mov x19, #0x0 // #0 : } 0.00 : ffff800010718770: ldr x25, [sp, #64] 0.00 : ffff800010718774: mov x0, x19 0.00 : ffff800010718778: ldp x19, x20, [sp, #16] 0.00 : ffff80001071877c: ldp x21, x22, [sp, #32] 0.00 : ffff800010718780: ldp x23, x24, [sp, #48] 0.00 : ffff800010718784: ldp x29, x30, [sp], #96 0.00 : ffff800010718788: ret : spin_unlock_irqrestore(): 0.00 : ffff80001071878c: mov x1, x27 0.00 : ffff800010718790: mov x0, x22 0.00 : ffff800010718794: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010718798: ldp x26, x27, [x29, #72] 0.00 : ffff80001071879c: b ffff8000107185b4 : iova_magazine_free(): : kfree(mag); 0.00 : ffff8000107187a0: ldr x0, [x22, #8] 0.00 : ffff8000107187a4: bl ffff800010253890 : __iova_rcache_get(): : cpu_rcache->loaded = rcache->depot[--rcache->depot_size]; 0.00 : ffff8000107187a8: ldr x2, [x28, #136] : spin_unlock(): : raw_spin_unlock(&lock->rlock); 0.00 : ffff8000107187ac: mov x0, x23 : __iova_rcache_get(): 0.00 : ffff8000107187b0: sub x2, x2, #0x1 0.00 : ffff8000107187b4: str x2, [x28, #136] 0.00 : ffff8000107187b8: add x19, x19, x2 0.00 : ffff8000107187bc: add x2, x19, #0x12 0.00 : ffff8000107187c0: ldr x1, [x21, x2, lsl #3] 0.00 : ffff8000107187c4: str x1, [x22, #8] : spin_unlock(): 0.00 : ffff8000107187c8: bl ffff800010cd7c38 <_raw_spin_unlock> : __iova_rcache_get(): : iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn); 0.00 : ffff8000107187cc: ldr x0, [x22, #8] : iova_magazine_empty(): : return (!mag || mag->size == 0); 0.00 : ffff8000107187d0: cbz x0, ffff8000107186c4 0.00 : ffff8000107187d4: ldr x5, [x0] 0.00 : ffff8000107187d8: b ffff8000107186c0 0.00 : ffff8000107187dc: mov w23, #0x118 // #280 0.00 : ffff8000107187e0: mov x0, #0x80 // #128 0.00 : ffff8000107187e4: stp x26, x27, [x29, #72] 0.00 : ffff8000107187e8: umaddl x23, w19, w23, x0 0.00 : ffff8000107187ec: b ffff800010718620 Percent | Source code & Disassembly of vmlinux for cycles (102 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010129378 : : dequeue_top_rt_rq(): : { : struct rq *rq = rq_of_rt_rq(rt_rq); : : BUG_ON(&rq->rt != rt_rq); : : if (!rt_rq->rt_queued) 27.54 : ffff800010129378: ldr w1, [x0, #1672] : BUG_ON(&rq->rt != rt_rq); 0.00 : ffff80001012937c: sub x2, x0, #0x200 : if (!rt_rq->rt_queued) 0.98 : ffff800010129380: cbz w1, ffff8000101293a0 : return; : : BUG_ON(!rq->nr_running); 15.91 : ffff800010129384: ldr w1, [x2, #4] 0.00 : ffff800010129388: cbz w1, ffff8000101293a4 : sub_nr_running(): : sched_update_tick_dependency(rq); : } : : static inline void sub_nr_running(struct rq *rq, unsigned count) : { : rq->nr_running -= count; 9.89 : ffff80001012938c: sub x2, x0, #0x4, lsl #12 4.92 : ffff800010129390: ldr w3, [x0, #1616] 0.00 : ffff800010129394: sub w1, w1, w3 1.98 : ffff800010129398: str w1, [x2, #15876] : dequeue_top_rt_rq(): : : sub_nr_running(rq, rt_rq->rt_nr_running); : rt_rq->rt_queued = 0; 1.96 : ffff80001012939c: str wzr, [x0, #1672] : : } 36.83 : ffff8000101293a0: ret : BUG_ON(!rq->nr_running); 0.00 : ffff8000101293a4: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (57 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016cff8 : : update_fast_timekeeper(): : * @ktime_get_mono_fast_ns. : */ : static void update_fast_timekeeper(const struct tk_read_base *tkr, : struct tk_fast *tkf) : { : struct tk_read_base *base = tkf->base; 0.00 : ffff80001016cff8: add x2, x1, #0x8 : raw_write_seqcount_latch(): : * NOTE: When data is a dynamic data structure; one should use regular RCU : * patterns to manage the lifetimes of the objects within. : */ : static inline void raw_write_seqcount_latch(seqcount_t *s) : { : smp_wmb(); /* prior stores before incrementing "sequence" */ 0.00 : ffff80001016cffc: dmb ishst : s->sequence++; 35.17 : ffff80001016d000: ldr w3, [x1] 0.00 : ffff80001016d004: add w3, w3, #0x1 5.21 : ffff80001016d008: str w3, [x1] : smp_wmb(); /* increment "sequence" before following stores */ 0.00 : ffff80001016d00c: dmb ishst : update_fast_timekeeper(): : : /* Force readers off to base[1] */ : raw_write_seqcount_latch(&tkf->seq); : : /* Update base[0] */ : memcpy(base, tkr, sizeof(*base)); 52.47 : ffff80001016d010: ldp x4, x5, [x0] 3.57 : ffff80001016d014: stp x4, x5, [x1, #8] 0.00 : ffff80001016d018: ldp x4, x5, [x0, #16] 0.00 : ffff80001016d01c: stp x4, x5, [x2, #16] 0.00 : ffff80001016d020: ldp x4, x5, [x0, #32] 0.00 : ffff80001016d024: stp x4, x5, [x2, #32] 0.00 : ffff80001016d028: ldr x0, [x0, #48] 0.00 : ffff80001016d02c: str x0, [x2, #48] : raw_write_seqcount_latch(): : smp_wmb(); /* prior stores before incrementing "sequence" */ 0.00 : ffff80001016d030: dmb ishst : s->sequence++; 0.00 : ffff80001016d034: mov x0, x1 0.00 : ffff80001016d038: ldr w3, [x1] 0.00 : ffff80001016d03c: add w3, w3, #0x1 0.00 : ffff80001016d040: str w3, [x0], #64 : smp_wmb(); /* increment "sequence" before following stores */ 0.00 : ffff80001016d044: dmb ishst : update_fast_timekeeper(): : : /* Force readers back to base[0] */ : raw_write_seqcount_latch(&tkf->seq); : : /* Update base[1] */ : memcpy(base + 1, base, sizeof(*base)); 3.59 : ffff80001016d048: ldp x4, x5, [x1, #8] 0.00 : ffff80001016d04c: stp x4, x5, [x1, #64] 0.00 : ffff80001016d050: ldp x4, x5, [x2, #16] 0.00 : ffff80001016d054: stp x4, x5, [x0, #16] 0.00 : ffff80001016d058: ldp x4, x5, [x2, #32] 0.00 : ffff80001016d05c: stp x4, x5, [x0, #32] 0.00 : ffff80001016d060: ldr x1, [x2, #48] 0.00 : ffff80001016d064: str x1, [x0, #48] : } 0.00 : ffff80001016d068: ret Percent | Source code & Disassembly of vmlinux for cycles (123 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001045b4c0 : : generic_make_request(): : * bio happens to be merged with someone else, and may resubmit the bio to : * a lower device by calling into generic_make_request recursively, which : * means the bio should NOT be touched after the call to ->make_request_fn. : */ : blk_qc_t generic_make_request(struct bio *bio) : { 0.00 : ffff80001045b4c0: stp x29, x30, [sp, #-128]! 0.00 : ffff80001045b4c4: mov x29, sp 0.00 : ffff80001045b4c8: stp x21, x22, [sp, #32] 0.00 : ffff80001045b4cc: adrp x21, ffff800011909000 0.81 : ffff80001045b4d0: str x26, [sp, #72] 0.00 : ffff80001045b4d4: add x1, x21, #0x908 0.00 : ffff80001045b4d8: mov x26, x0 0.84 : ffff80001045b4dc: ldr x2, [x1] 0.00 : ffff80001045b4e0: str x2, [x29, #120] 0.00 : ffff80001045b4e4: mov x2, #0x0 // #0 : * yet. : */ : struct bio_list bio_list_on_stack[2]; : blk_qc_t ret = BLK_QC_T_NONE; : : if (!generic_make_request_checks(bio)) 0.00 : ffff80001045b4e8: bl ffff80001045ad10 0.00 : ffff80001045b4ec: tst w0, #0xff 0.00 : ffff80001045b4f0: b.eq ffff80001045b764 // b.none : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 1.59 : ffff80001045b4f4: mrs x1, sp_el0 : generic_make_request(): : * flag to say if generic_make_request is currently active in this : * task or not. If it is NULL, then no make_request is active. If : * it is non-NULL, then a make_request is active, and new requests : * should be added at the tail : */ : if (current->bio_list) { 0.00 : ffff80001045b4f8: ldr x0, [x1, #1832] 0.00 : ffff80001045b4fc: cbz x0, ffff80001045b540 : bio_list_add(): : return sz; : } : : static inline void bio_list_add(struct bio_list *bl, struct bio *bio) : { : bio->bi_next = NULL; 0.00 : ffff80001045b500: str xzr, [x26] : : if (bl->tail) 0.00 : ffff80001045b504: ldr x1, [x0, #8] 0.00 : ffff80001045b508: cbz x1, ffff80001045b7a8 : bl->tail->bi_next = bio; 0.00 : ffff80001045b50c: str x26, [x1] : generic_make_request(): : blk_qc_t ret = BLK_QC_T_NONE; 0.00 : ffff80001045b510: mov w22, #0xffffffff // #-1 : bio_list_add(): : else : bl->head = bio; : : bl->tail = bio; 0.00 : ffff80001045b514: str x26, [x0, #8] : generic_make_request(): : } while (bio); : current->bio_list = NULL; /* deactivate */ : : out: : return ret; : } 0.00 : ffff80001045b518: add x21, x21, #0x908 0.00 : ffff80001045b51c: mov w0, w22 0.82 : ffff80001045b520: ldr x2, [x29, #120] 1.64 : ffff80001045b524: ldr x1, [x21] 0.00 : ffff80001045b528: eor x1, x2, x1 0.00 : ffff80001045b52c: cbnz x1, ffff80001045b7d0 0.00 : ffff80001045b530: ldp x21, x22, [sp, #32] 0.00 : ffff80001045b534: ldr x26, [sp, #72] 1.66 : ffff80001045b538: ldp x29, x30, [sp], #128 0.00 : ffff80001045b53c: ret : BUG_ON(bio->bi_next); 0.84 : ffff80001045b540: ldr x0, [x26] 0.82 : ffff80001045b544: stp x19, x20, [x29, #16] 0.81 : ffff80001045b548: stp x23, x24, [x29, #48] 0.00 : ffff80001045b54c: str x25, [x29, #64] 0.00 : ffff80001045b550: cbnz x0, ffff80001045b7cc : bio_list_init(): : bl->head = bl->tail = NULL; 1.66 : ffff80001045b554: add x0, x29, #0x80 0.00 : ffff80001045b558: str xzr, [x29, #96] : generic_make_request(): : blk_qc_t ret = BLK_QC_T_NONE; 0.00 : ffff80001045b55c: mov w22, #0xffffffff // #-1 : bio_io_error(): : bio->bi_status = BLK_STS_IOERR; 0.00 : ffff80001045b560: mov w24, #0xa // #10 : bio_wouldblock_error(): : bio->bi_status = BLK_STS_AGAIN; 0.00 : ffff80001045b564: mov w25, #0xc // #12 : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045b568: mov x20, #0xffffffffffffffff // #-1 : bio_list_init(): : bl->head = bl->tail = NULL; 0.00 : ffff80001045b56c: str xzr, [x0, #-40]! : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff80001045b570: mov x23, #0x1 // #1 : generic_make_request(): : current->bio_list = bio_list_on_stack; 0.00 : ffff80001045b574: str x0, [x1, #1832] : struct request_queue *q = bio->bi_disk->queue; 0.82 : ffff80001045b578: ldr x0, [x26, #8] : BLK_MQ_REQ_NOWAIT : 0; 0.00 : ffff80001045b57c: ldr w1, [x26, #16] : struct request_queue *q = bio->bi_disk->queue; 0.82 : ffff80001045b580: ldr x19, [x0, #1040] : if (likely(blk_queue_enter(q, flags) == 0)) { 0.00 : ffff80001045b584: ubfx w1, w1, #21, #1 0.00 : ffff80001045b588: mov x0, x19 0.00 : ffff80001045b58c: bl ffff80001045b1f8 5.59 : ffff80001045b590: cbnz w0, ffff80001045b6d0 0.82 : ffff80001045b594: nop : bio_list_on_stack[1] = bio_list_on_stack[0]; 0.00 : ffff80001045b598: ldp x2, x3, [x29, #88] 26.04 : ffff80001045b59c: stp x2, x3, [x29, #104] : ret = q->make_request_fn(q, bio); 0.00 : ffff80001045b5a0: mov x1, x26 : bio_list_init(): 0.00 : ffff80001045b5a4: stp xzr, xzr, [x29, #88] : generic_make_request(): 0.00 : ffff80001045b5a8: ldr x2, [x19, #32] 0.00 : ffff80001045b5ac: mov x0, x19 0.00 : ffff80001045b5b0: blr x2 0.00 : ffff80001045b5b4: mov w22, w0 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001045b5b8: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001045b5bc: ldr x0, [x19, #1480] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001045b5c0: tst x0, #0x3 0.00 : ffff80001045b5c4: b.ne ffff80001045b76c // b.any : get_current(): 0.00 : ffff80001045b5c8: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff80001045b5cc: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001045b5d0: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 3.18 : ffff80001045b5d4: str w2, [x1, #16] : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001045b5d8: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045b5dc: add x0, x0, x2 0.00 : ffff80001045b5e0: ldxr x4, [x0] 23.51 : ffff80001045b5e4: add x4, x4, x20 0.00 : ffff80001045b5e8: stxr w3, x4, [x0] 0.00 : ffff80001045b5ec: cbnz w3, ffff80001045b5e0 : __read_once_size(): : __READ_ONCE_SIZE; 1.65 : ffff80001045b5f0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045b5f4: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.79 : ffff80001045b5f8: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045b5fc: cbz x0, ffff80001045b710 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045b600: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045b604: cbz x0, ffff80001045b710 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 5.74 : ffff80001045b608: bl ffff800010160d20 <__rcu_read_unlock> : bio_list_init(): 0.00 : ffff80001045b60c: mov x3, #0x0 // #0 0.00 : ffff80001045b610: mov x5, #0x0 // #0 0.00 : ffff80001045b614: mov x2, #0x0 // #0 0.00 : ffff80001045b618: mov x4, #0x0 // #0 : bio_list_pop(): : return bl->head; : } : : static inline struct bio *bio_list_pop(struct bio_list *bl) : { : struct bio *bio = bl->head; 1.67 : ffff80001045b61c: ldr x0, [x29, #88] : : if (bio) { 0.00 : ffff80001045b620: cbz x0, ffff80001045b658 : bl->head = bl->head->bi_next; 0.00 : ffff80001045b624: ldr x1, [x0] 0.00 : ffff80001045b628: str x1, [x29, #88] : if (!bl->head) 0.00 : ffff80001045b62c: cbz x1, ffff80001045b728 : generic_make_request(): : if (q == bio->bi_disk->queue) 0.00 : ffff80001045b630: ldr x1, [x0, #8] : bio_list_pop(): : bl->tail = NULL; : : bio->bi_next = NULL; 0.00 : ffff80001045b634: str xzr, [x0] : generic_make_request(): 0.00 : ffff80001045b638: ldr x1, [x1, #1040] 0.00 : ffff80001045b63c: cmp x19, x1 0.00 : ffff80001045b640: b.eq ffff80001045b718 // b.none : bio_list_add(): : if (bl->tail) 0.00 : ffff80001045b644: cbz x2, ffff80001045b730 : bl->tail->bi_next = bio; 0.00 : ffff80001045b648: str x0, [x2] 0.00 : ffff80001045b64c: mov x2, x0 : bio_list_pop(): : struct bio *bio = bl->head; 0.00 : ffff80001045b650: ldr x0, [x29, #88] : if (bio) { 0.00 : ffff80001045b654: cbnz x0, ffff80001045b624 : bio_list_merge(): : if (!bl2->head) 0.00 : ffff80001045b658: cbz x4, ffff80001045b66c : if (bl->tail) 0.00 : ffff80001045b65c: ldr x0, [x29, #96] 0.00 : ffff80001045b660: cbz x0, ffff80001045b758 : bl->tail->bi_next = bl2->head; 0.00 : ffff80001045b664: str x4, [x0] : bl->tail = bl2->tail; 0.00 : ffff80001045b668: str x2, [x29, #96] : if (!bl2->head) 0.00 : ffff80001045b66c: cbz x5, ffff80001045b680 : if (bl->tail) 0.00 : ffff80001045b670: ldr x0, [x29, #96] 0.00 : ffff80001045b674: cbz x0, ffff80001045b750 : bl->tail->bi_next = bl2->head; 0.00 : ffff80001045b678: str x5, [x0] : bl->tail = bl2->tail; 0.00 : ffff80001045b67c: str x3, [x29, #96] : if (!bl2->head) 0.78 : ffff80001045b680: ldr x26, [x29, #104] 0.00 : ffff80001045b684: cbz x26, ffff80001045b6e4 : if (bl->tail) 0.00 : ffff80001045b688: ldr x0, [x29, #96] 0.00 : ffff80001045b68c: cbz x0, ffff80001045b748 : bl->tail->bi_next = bl2->head; 0.00 : ffff80001045b690: str x26, [x0] 0.00 : ffff80001045b694: ldr x26, [x29, #88] : bl->tail = bl2->tail; 0.00 : ffff80001045b698: ldr x0, [x29, #112] 0.00 : ffff80001045b69c: str x0, [x29, #96] : bio_list_pop(): : if (bio) { 0.00 : ffff80001045b6a0: cbz x26, ffff80001045b6ec : bl->head = bl->head->bi_next; 0.00 : ffff80001045b6a4: ldr x0, [x26] 0.00 : ffff80001045b6a8: str x0, [x29, #88] : if (!bl->head) 0.00 : ffff80001045b6ac: cbz x0, ffff80001045b704 : bio->bi_next = NULL; 0.00 : ffff80001045b6b0: str xzr, [x26] : generic_make_request(): : struct request_queue *q = bio->bi_disk->queue; 0.00 : ffff80001045b6b4: ldr x0, [x26, #8] : BLK_MQ_REQ_NOWAIT : 0; 0.00 : ffff80001045b6b8: ldr w1, [x26, #16] : struct request_queue *q = bio->bi_disk->queue; 0.00 : ffff80001045b6bc: ldr x19, [x0, #1040] : if (likely(blk_queue_enter(q, flags) == 0)) { 0.00 : ffff80001045b6c0: ubfx w1, w1, #21, #1 0.00 : ffff80001045b6c4: mov x0, x19 0.00 : ffff80001045b6c8: bl ffff80001045b1f8 0.00 : ffff80001045b6cc: cbz w0, ffff80001045b598 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001045b6d0: ldr x0, [x19, #104] : generic_make_request(): : if (unlikely(!blk_queue_dying(q) && 0.00 : ffff80001045b6d4: tbz w0, #1, ffff80001045b7b0 : bio_io_error(): : bio->bi_status = BLK_STS_IOERR; 0.00 : ffff80001045b6d8: strb w24, [x26, #26] : bio_endio(bio); 0.00 : ffff80001045b6dc: mov x0, x26 0.00 : ffff80001045b6e0: bl ffff800010456b18 4.93 : ffff80001045b6e4: ldr x26, [x29, #88] : bio_list_pop(): : if (bio) { 0.00 : ffff80001045b6e8: cbnz x26, ffff80001045b6a4 : get_current(): 6.46 : ffff80001045b6ec: mrs x0, sp_el0 : generic_make_request(): : current->bio_list = NULL; /* deactivate */ 0.00 : ffff80001045b6f0: str xzr, [x0, #1832] 5.71 : ffff80001045b6f4: ldp x19, x20, [x29, #16] 0.00 : ffff80001045b6f8: ldp x23, x24, [x29, #48] 0.00 : ffff80001045b6fc: ldr x25, [x29, #64] 0.00 : ffff80001045b700: b ffff80001045b518 : bio_list_pop(): : bl->tail = NULL; 0.00 : ffff80001045b704: str xzr, [x29, #96] : bio->bi_next = NULL; 0.00 : ffff80001045b708: str xzr, [x26] 0.00 : ffff80001045b70c: b ffff80001045b6b4 : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff80001045b710: bl ffff800010cd2e78 0.00 : ffff80001045b714: b ffff80001045b608 : bio_list_add(): : if (bl->tail) 0.00 : ffff80001045b718: cbz x3, ffff80001045b73c : bl->tail->bi_next = bio; 0.00 : ffff80001045b71c: str x0, [x3] 0.00 : ffff80001045b720: mov x3, x0 0.00 : ffff80001045b724: b ffff80001045b61c : bio_list_pop(): : bl->tail = NULL; 0.00 : ffff80001045b728: str xzr, [x29, #96] 0.00 : ffff80001045b72c: b ffff80001045b630 : bio_list_add(): : if (bl->tail) 0.00 : ffff80001045b730: mov x2, x0 0.00 : ffff80001045b734: mov x4, x0 0.00 : ffff80001045b738: b ffff80001045b61c 0.00 : ffff80001045b73c: mov x3, x0 0.00 : ffff80001045b740: mov x5, x0 0.00 : ffff80001045b744: b ffff80001045b61c : bio_list_merge(): : bl->head = bl2->head; 0.00 : ffff80001045b748: str x26, [x29, #88] 0.00 : ffff80001045b74c: b ffff80001045b698 0.00 : ffff80001045b750: str x5, [x29, #88] 0.00 : ffff80001045b754: b ffff80001045b67c 0.00 : ffff80001045b758: str x4, [x29, #88] : bl->tail = bl2->tail; 0.00 : ffff80001045b75c: str x2, [x29, #96] 0.00 : ffff80001045b760: b ffff80001045b66c : generic_make_request(): : blk_qc_t ret = BLK_QC_T_NONE; 0.00 : ffff80001045b764: mov w22, #0xffffffff // #-1 : return ret; 0.00 : ffff80001045b768: b ffff80001045b518 : blk_queue_exit(): : percpu_ref_put(&q->q_usage_counter); 0.00 : ffff80001045b76c: add x0, x19, #0x5c0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001045b770: b ffff80001045b798 0.00 : ffff80001045b774: b ffff80001045b798 : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001045b778: mov x1, x23 0.00 : ffff80001045b77c: neg x1, x1 0.00 : ffff80001045b780: ldaddal x1, x2, [x0] 0.00 : ffff80001045b784: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001045b788: cbnz x1, ffff80001045b608 : ref->release(ref); 0.00 : ffff80001045b78c: ldr x1, [x0, #16] 0.00 : ffff80001045b790: blr x1 0.00 : ffff80001045b794: b ffff80001045b608 : __ll_sc_atomic64_sub_return(): 0.00 : ffff80001045b798: add x3, x19, #0x5c0 0.00 : ffff80001045b79c: b ffff80001045ce7c : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001045b7a0: cbnz x1, ffff80001045b608 0.00 : ffff80001045b7a4: b ffff80001045b78c : bio_list_add(): : bl->head = bio; 0.00 : ffff80001045b7a8: str x26, [x0] 0.00 : ffff80001045b7ac: b ffff80001045b510 : generic_make_request(): : if (unlikely(!blk_queue_dying(q) && 0.00 : ffff80001045b7b0: ldr w0, [x26, #16] 0.00 : ffff80001045b7b4: tbz w0, #21, ffff80001045b6d8 : bio_wouldblock_error(): : bio->bi_status = BLK_STS_AGAIN; 0.00 : ffff80001045b7b8: strb w25, [x26, #26] : bio_endio(bio); 0.00 : ffff80001045b7bc: mov x0, x26 0.00 : ffff80001045b7c0: bl ffff800010456b18 0.00 : ffff80001045b7c4: ldr x26, [x29, #88] 0.00 : ffff80001045b7c8: b ffff80001045b6a0 : generic_make_request(): : BUG_ON(bio->bi_next); 0.00 : ffff80001045b7cc: brk #0x800 0.00 : ffff80001045b7d0: stp x19, x20, [x29, #16] 0.00 : ffff80001045b7d4: stp x23, x24, [x29, #48] 0.00 : ffff80001045b7d8: str x25, [x29, #64] : } 0.00 : ffff80001045b7dc: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (54 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101524d8 : : handle_percpu_devid_irq(): : * action->percpu_dev_id is a pointer to percpu variables which : * contain the real device id for the cpu on which this handler is : * called : */ : void handle_percpu_devid_irq(struct irq_desc *desc) : { 22.72 : ffff8000101524d8: stp x29, x30, [sp, #-48]! 0.00 : ffff8000101524dc: mov x29, sp 13.09 : ffff8000101524e0: stp x19, x20, [sp, #16] 0.00 : ffff8000101524e4: mov x19, x0 0.00 : ffff8000101524e8: stp x21, x22, [sp, #32] : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff8000101524ec: mov x3, sp : asm(ALTERNATIVE("mrs %0, tpidr_el1", 1.89 : ffff8000101524f0: mrs x4, tpidr_el1 : __kstat_incr_irqs_this_cpu(): : : #undef __irqd_to_state : : static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc) : { : __this_cpu_inc(*desc->kstat_irqs); 0.00 : ffff8000101524f4: ldr x1, [x19, #144] : __this_cpu_inc(kstat.irqs_sum); 0.00 : ffff8000101524f8: adrp x0, ffff80001151f000 : __this_cpu_inc(*desc->kstat_irqs); 0.00 : ffff8000101524fc: ldr w2, [x1, x4] : __this_cpu_inc(kstat.irqs_sum); 3.68 : ffff800010152500: add x0, x0, #0x10 : handle_percpu_devid_irq(): : struct irq_chip *chip = irq_desc_get_chip(desc); 17.26 : ffff800010152504: ldr x20, [x19, #112] : struct irqaction *action = desc->action; : unsigned int irq = irq_desc_get_irq(desc); 0.00 : ffff800010152508: ldr w21, [x19, #92] : __kstat_incr_irqs_this_cpu(): : __this_cpu_inc(*desc->kstat_irqs); 0.00 : ffff80001015250c: add w2, w2, #0x1 : handle_percpu_devid_irq(): : struct irqaction *action = desc->action; 0.00 : ffff800010152510: ldr x22, [x19, #160] : __kstat_incr_irqs_this_cpu(): 0.00 : ffff800010152514: str w2, [x1, x4] : __my_cpu_offset(): 3.72 : ffff800010152518: mrs x2, tpidr_el1 : __kstat_incr_irqs_this_cpu(): : __this_cpu_inc(kstat.irqs_sum); 0.00 : ffff80001015251c: ldr x1, [x0, x2] 0.00 : ffff800010152520: add x1, x1, #0x1 0.00 : ffff800010152524: str x1, [x0, x2] : handle_percpu_devid_irq(): : * PER CPU interrupts are not serialized. Do not touch : * desc->tot_count. : */ : __kstat_incr_irqs_this_cpu(desc); : : if (chip->irq_ack) 0.00 : ffff800010152528: ldr x1, [x20, #48] 0.00 : ffff80001015252c: cbz x1, ffff800010152538 : chip->irq_ack(&desc->irq_data); 0.00 : ffff800010152530: add x0, x19, #0x58 0.00 : ffff800010152534: blr x1 1.96 : ffff800010152538: mov x0, sp : : if (likely(action)) { 0.00 : ffff80001015253c: cbz x22, ffff800010152578 : trace_irq_handler_entry(irq, action); : res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id)); 0.00 : ffff800010152540: ldr x2, [x22] : __my_cpu_offset(): 11.43 : ffff800010152544: mrs x3, tpidr_el1 : handle_percpu_devid_irq(): 0.00 : ffff800010152548: ldr x1, [x22, #16] 0.00 : ffff80001015254c: mov w0, w21 0.00 : ffff800010152550: add x1, x1, x3 0.00 : ffff800010152554: blr x2 : : pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n", : enabled ? " and unmasked" : "", irq, cpu); : } : : if (chip->irq_eoi) 0.00 : ffff800010152558: ldr x1, [x20, #80] 0.00 : ffff80001015255c: cbz x1, ffff800010152568 : chip->irq_eoi(&desc->irq_data); 3.42 : ffff800010152560: add x0, x19, #0x58 0.00 : ffff800010152564: blr x1 : } 1.89 : ffff800010152568: ldp x19, x20, [sp, #16] 18.93 : ffff80001015256c: ldp x21, x22, [sp, #32] 0.00 : ffff800010152570: ldp x29, x30, [sp], #48 0.00 : ffff800010152574: ret : __my_cpu_offset(): 0.00 : ffff800010152578: mrs x0, tpidr_el1 : handle_percpu_devid_irq(): : unsigned int cpu = smp_processor_id(); 0.00 : ffff80001015257c: adrp x1, ffff80001151d000 0.00 : ffff800010152580: add x1, x1, #0x18 0.00 : ffff800010152584: ldr w22, [x1, x0] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010152588: ldr x2, [x19, #216] 0.00 : ffff80001015258c: add w0, w22, #0x3f 0.00 : ffff800010152590: cmp w22, #0x0 0.00 : ffff800010152594: csel w0, w0, w22, lt // lt = tstop 0.00 : ffff800010152598: asr w0, w0, #6 0.00 : ffff80001015259c: sxtw x0, w0 0.00 : ffff8000101525a0: ldr x0, [x2, x0, lsl #3] 0.00 : ffff8000101525a4: lsr x0, x0, x22 : handle_percpu_devid_irq(): : if (enabled) 0.00 : ffff8000101525a8: tbnz w0, #0, ffff8000101525d0 : pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n", 0.00 : ffff8000101525ac: adrp x0, ffff800011ad8000 0.00 : ffff8000101525b0: add x0, x0, #0x4c0 0.00 : ffff8000101525b4: ldrb w1, [x0, #1] 0.00 : ffff8000101525b8: cbnz w1, ffff800010152558 0.00 : ffff8000101525bc: mov w2, #0x1 // #1 0.00 : ffff8000101525c0: adrp x1, ffff8000111d8000 0.00 : ffff8000101525c4: strb w2, [x0, #1] 0.00 : ffff8000101525c8: add x1, x1, #0x78 0.00 : ffff8000101525cc: b ffff8000101525fc : irq_percpu_disable(desc, cpu); 0.00 : ffff8000101525d0: mov w1, w22 0.00 : ffff8000101525d4: mov x0, x19 0.00 : ffff8000101525d8: bl ffff800010152360 : pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n", 0.00 : ffff8000101525dc: adrp x0, ffff800011ad8000 0.00 : ffff8000101525e0: add x0, x0, #0x4c0 0.00 : ffff8000101525e4: ldrb w1, [x0, #1] 0.00 : ffff8000101525e8: cbnz w1, ffff800010152558 0.00 : ffff8000101525ec: adrp x1, ffff8000111c6000 0.00 : ffff8000101525f0: add x1, x1, #0xa88 0.00 : ffff8000101525f4: mov w2, #0x1 // #1 0.00 : ffff8000101525f8: strb w2, [x0, #1] 0.00 : ffff8000101525fc: adrp x0, ffff8000111c6000 0.00 : ffff800010152600: mov w3, w22 0.00 : ffff800010152604: mov w2, w21 0.00 : ffff800010152608: add x0, x0, #0xa98 0.00 : ffff80001015260c: bl ffff80001014a85c 0.00 : ffff800010152610: b ffff800010152558 Percent | Source code & Disassembly of vmlinux for cycles (143 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102a1428 <__fget_files>: : __fget_files(): : spin_unlock(&files->file_lock); : } : : static struct file *__fget_files(struct files_struct *files, unsigned int fd, : fmode_t mask, unsigned int refs) : { 0.71 : ffff8000102a1428: stp x29, x30, [sp, #-48]! 0.00 : ffff8000102a142c: mov x29, sp 0.69 : ffff8000102a1430: stp x19, x20, [sp, #16] 0.00 : ffff8000102a1434: mov w20, w1 0.00 : ffff8000102a1438: mov w19, w3 0.00 : ffff8000102a143c: stp x21, x22, [sp, #32] 0.00 : ffff8000102a1440: mov x21, x0 0.00 : ffff8000102a1444: mov w22, w2 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.66 : ffff8000102a1448: bl ffff80001015c1b0 <__rcu_read_lock> : __fcheck_files(): : static inline struct file *__fcheck_files(struct files_struct *files, unsigned int fd) : { : struct fdtable *fdt = rcu_dereference_raw(files->fdt); : : if (fd < fdt->max_fds) { : fd = array_index_nospec(fd, fdt->max_fds); 3.50 : ffff8000102a144c: mov w6, w20 : __fget_files(): : * dup2() atomicity guarantee is the reason : * we loop to catch the new file (or NULL pointer) : */ : if (file->f_mode & mask) : file = NULL; : else if (!get_file_rcu_many(file, refs)) 0.00 : ffff8000102a1450: mov w3, w19 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000102a1454: ldr x1, [x21, #32] : __fcheck_files(): : if (fd < fdt->max_fds) { 1.42 : ffff8000102a1458: ldr w0, [x1] 0.00 : ffff8000102a145c: cmp w20, w0 0.00 : ffff8000102a1460: b.cs ffff8000102a14f4 <__fget_files+0xcc> // b.hs, b.nlast : fd = array_index_nospec(fd, fdt->max_fds); 3.55 : ffff8000102a1464: mov w0, w0 : array_index_mask_nospec(): : static inline unsigned long array_index_mask_nospec(unsigned long idx, : unsigned long sz) : { : unsigned long mask; : : asm volatile( 0.00 : ffff8000102a1468: cmp x6, x0 0.00 : ffff8000102a146c: ngc x0, xzr : " sbc %0, xzr, xzr\n" : : "=r" (mask) : : "r" (idx), "Ir" (sz) : : "cc"); : : csdb(); 0.00 : ffff8000102a1470: csdb : __fcheck_files(): : return rcu_dereference_raw(fdt->fd[fd]); 0.71 : ffff8000102a1474: and w0, w20, w0 0.00 : ffff8000102a1478: ldr x1, [x1, #8] : __read_once_size(): 1.43 : ffff8000102a147c: ldr x19, [x1, x0, lsl #3] : __fget_files(): : if (file) { 0.00 : ffff8000102a1480: cbz x19, ffff8000102a14f4 <__fget_files+0xcc> : if (file->f_mode & mask) 14.52 : ffff8000102a1484: ldr w0, [x19, #68] 0.00 : ffff8000102a1488: tst w22, w0 0.00 : ffff8000102a148c: b.ne ffff8000102a14f4 <__fget_files+0xcc> // b.any : __read_once_size(): 18.98 : ffff8000102a1490: ldr x4, [x19, #56] 0.00 : ffff8000102a1494: add x5, x19, #0x38 : atomic64_fetch_add_unless(): : atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) : { : s64 c = atomic64_read(v); : : do { : if (unlikely(c == u)) 0.00 : ffff8000102a1498: cbz x4, ffff8000102a1454 <__fget_files+0x2c> : break; : } while (!atomic64_try_cmpxchg(v, &c, c + a)); 2.13 : ffff8000102a149c: add x2, x3, x4 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102a14a0: b ffff8000102a14dc <__fget_files+0xb4> 1.41 : ffff8000102a14a4: b ffff8000102a14dc <__fget_files+0xb4> : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff8000102a14a8: mov x0, x5 0.00 : ffff8000102a14ac: mov x1, x4 0.00 : ffff8000102a14b0: mov x7, x1 0.00 : ffff8000102a14b4: casal x7, x2, [x5] 48.85 : ffff8000102a14b8: mov x0, x7 : atomic64_try_cmpxchg(): : if (unlikely(r != o)) 0.00 : ffff8000102a14bc: cmp x0, x4 0.00 : ffff8000102a14c0: b.ne ffff8000102a14e8 <__fget_files+0xc0> // b.any : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff8000102a14c4: bl ffff800010160d20 <__rcu_read_unlock> : __fget_files(): : goto loop; : } : rcu_read_unlock(); : : return file; : } 0.00 : ffff8000102a14c8: mov x0, x19 1.42 : ffff8000102a14cc: ldp x19, x20, [sp, #16] 0.00 : ffff8000102a14d0: ldp x21, x22, [sp, #32] 0.00 : ffff8000102a14d4: ldp x29, x30, [sp], #48 0.00 : ffff8000102a14d8: ret : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff8000102a14dc: b ffff8000102a2a20 : atomic64_try_cmpxchg(): 0.00 : ffff8000102a14e0: cmp x0, x4 0.00 : ffff8000102a14e4: b.eq ffff8000102a14c4 <__fget_files+0x9c> // b.none 0.00 : ffff8000102a14e8: mov x4, x0 : atomic64_fetch_add_unless(): : if (unlikely(c == u)) 0.00 : ffff8000102a14ec: cbnz x0, ffff8000102a149c <__fget_files+0x74> 0.00 : ffff8000102a14f0: b ffff8000102a1454 <__fget_files+0x2c> : rcu_read_unlock(): 0.00 : ffff8000102a14f4: bl ffff800010160d20 <__rcu_read_unlock> : __fcheck_files(): : } : return NULL; 0.00 : ffff8000102a14f8: mov x19, #0x0 // #0 : __fget_files(): 0.00 : ffff8000102a14fc: ldp x21, x22, [sp, #32] 0.00 : ffff8000102a1500: mov x0, x19 0.00 : ffff8000102a1504: ldp x19, x20, [sp, #16] 0.00 : ffff8000102a1508: ldp x29, x30, [sp], #48 0.00 : ffff8000102a150c: ret Percent | Source code & Disassembly of vmlinux for cycles (63 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101261a8 : : load_balance(): : * tasks if there is an imbalance. : */ : static int load_balance(int this_cpu, struct rq *this_rq, : struct sched_domain *sd, enum cpu_idle_type idle, : int *continue_balancing) : { 0.00 : ffff8000101261a8: stp x29, x30, [sp, #-256]! 0.00 : ffff8000101261ac: mov x6, x2 0.00 : ffff8000101261b0: mov x29, sp 0.00 : ffff8000101261b4: str x21, [sp, #32] : int ld_moved, cur_ld_moved, active_balance = 0; : struct sched_domain *sd_parent = sd->parent; : struct sched_group *group; : struct rq *busiest; : struct rq_flags rf; : struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 0.00 : ffff8000101261b8: adrp x21, ffff80001151f000 : { 0.00 : ffff8000101261bc: str x26, [sp, #72] : struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 0.00 : ffff8000101261c0: add x21, x21, #0x58 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000101261c4: mrs x5, tpidr_el1 : load_balance(): 0.00 : ffff8000101261c8: add x21, x21, #0x20 : { 0.00 : ffff8000101261cc: str x2, [x29, #128] : struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 0.00 : ffff8000101261d0: add x7, x21, x5 : { 0.00 : ffff8000101261d4: adrp x5, ffff800011909000 0.00 : ffff8000101261d8: add x5, x5, #0x908 : : struct lb_env env = { 0.00 : ffff8000101261dc: str x2, [x29, #144] 0.00 : ffff8000101261e0: str x1, [x29, #168] : { 0.00 : ffff8000101261e4: ldr x1, [x5] 0.00 : ffff8000101261e8: str x1, [x29, #248] 0.00 : ffff8000101261ec: mov x1, #0x0 // #0 : sched_group_span(): : unsigned long cpumask[0]; : }; : : static inline struct cpumask *sched_group_span(struct sched_group *sg) : { : return to_cpumask(sg->cpumask); 0.00 : ffff8000101261f0: ldr x2, [x2, #16] : load_balance(): : struct sched_domain *sd_parent = sd->parent; 0.00 : ffff8000101261f4: ldr x1, [x6] : struct lb_env env = { 0.00 : ffff8000101261f8: stp xzr, xzr, [x29, #152] : sched_group_span(): 0.00 : ffff8000101261fc: add x2, x2, #0x20 : load_balance(): : { 0.00 : ffff800010126200: stp w0, w3, [x29, #104] : struct sched_domain *sd_parent = sd->parent; 0.00 : ffff800010126204: str x1, [x29, #120] : struct lb_env env = { 0.00 : ffff800010126208: mov w1, #0x2 // #2 0.00 : ffff80001012620c: str w0, [x29, #164] 0.00 : ffff800010126210: add x0, x29, #0xe8 0.00 : ffff800010126214: str x2, [x29, #176] 0.00 : ffff800010126218: mov w2, #0x20 // #32 0.00 : ffff80001012621c: stp xzr, xzr, [x29, #184] 0.00 : ffff800010126220: stp xzr, xzr, [x29, #200] 0.00 : ffff800010126224: stp xzr, xzr, [x29, #216] : struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 6.56 : ffff800010126228: str x7, [x29, #96] : { 2.88 : ffff80001012622c: str x4, [x29, #112] : struct lb_env env = { 0.00 : ffff800010126230: str w3, [x29, #188] : bitmap_and(): : static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, : const unsigned long *src2, unsigned int nbits) : { : if (small_const_nbits(nbits)) : return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0; : return __bitmap_and(dst, src1, src2, nbits); 0.00 : ffff800010126234: mov w3, #0x100 // #256 : load_balance(): 0.00 : ffff800010126238: str x7, [x29, #200] 0.00 : ffff80001012623c: str w2, [x29, #216] : bitmap_and(): 0.00 : ffff800010126240: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> : load_balance(): 0.00 : ffff800010126244: str w1, [x29, #224] : bitmap_and(): 0.00 : ffff800010126248: add x2, x2, #0x1c8 : load_balance(): 0.00 : ffff80001012624c: stp x0, x0, [x29, #232] : bitmap_and(): 0.00 : ffff800010126250: add x1, x6, #0x88 0.00 : ffff800010126254: mov x0, x7 0.00 : ffff800010126258: bl ffff8000104850b8 <__bitmap_and> : should_we_balance(): : if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 0.00 : ffff80001012625c: ldr w1, [x29, #164] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010126260: ldr x2, [x29, #200] 0.00 : ffff800010126264: add w0, w1, #0x3f 0.00 : ffff800010126268: cmp w1, #0x0 0.00 : ffff80001012626c: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff800010126270: asr w0, w0, #6 0.00 : ffff800010126274: sxtw x0, w0 0.00 : ffff800010126278: ldr x0, [x2, x0, lsl #3] 0.00 : ffff80001012627c: lsr x1, x0, x1 : should_we_balance(): 0.00 : ffff800010126280: tbz w1, #0, ffff8000101268e8 : struct sched_group *sg = env->sd->groups; 0.00 : ffff800010126284: ldr x0, [x29, #144] 0.00 : ffff800010126288: stp x19, x20, [x29, #16] : find_busiest_queue(): : rq = cpu_rq(i); 0.00 : ffff80001012628c: adrp x20, ffff800011528000 0.00 : ffff800010126290: stp x22, x23, [x29, #40] 0.00 : ffff800010126294: add x20, x20, #0xe80 0.00 : ffff800010126298: stp x24, x25, [x29, #56] 0.00 : ffff80001012629c: stp x27, x28, [x29, #80] : should_we_balance(): : struct sched_group *sg = env->sd->groups; 0.00 : ffff8000101262a0: ldr x19, [x0, #16] : if (env->idle == CPU_NEWLY_IDLE) 0.00 : ffff8000101262a4: ldr w0, [x29, #188] 0.00 : ffff8000101262a8: cmp w0, #0x2 0.00 : ffff8000101262ac: b.eq ffff8000101262f8 // b.none 0.00 : ffff8000101262b0: adrp x5, ffff80001190a000 <__per_cpu_offset+0x6d8> : for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) { 0.00 : ffff8000101262b4: add x22, x5, #0x2f0 0.00 : ffff8000101262b8: mov w23, #0xffffffff // #-1 : group_balance_mask(): : /* : * See build_balance_mask(). : */ : static inline struct cpumask *group_balance_mask(struct sched_group *sg) : { : return to_cpumask(sg->sgc->cpumask); 0.00 : ffff8000101262bc: ldr x1, [x19, #16] : should_we_balance(): 0.00 : ffff8000101262c0: mov w0, w23 0.00 : ffff8000101262c4: add x1, x1, #0x30 1.33 : ffff8000101262c8: bl ffff800010cb93f0 0.00 : ffff8000101262cc: ldr w1, [x22] 0.00 : ffff8000101262d0: mov w23, w0 0.00 : ffff8000101262d4: cmp w0, w1 0.00 : ffff8000101262d8: b.cs ffff800010126ba0 // b.hs, b.nlast : if (!idle_cpu(cpu)) 0.00 : ffff8000101262dc: bl ffff800010118820 0.00 : ffff8000101262e0: cbz w0, ffff800010126b04 : if (balance_cpu == -1) 0.00 : ffff8000101262e4: cmn w23, #0x1 0.00 : ffff8000101262e8: b.eq ffff800010126ba0 // b.none : load_balance(): : cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); : : schedstat_inc(sd->lb_count[idle]); : : redo: : if (!should_we_balance(&env)) { 1.95 : ffff8000101262ec: ldr w0, [x29, #164] 0.00 : ffff8000101262f0: cmp w0, w23 0.00 : ffff8000101262f4: b.ne ffff8000101268d8 // b.any : *continue_balancing = 0; : goto out_balanced; : } : : group = find_busiest_group(&env); 1.08 : ffff8000101262f8: add x0, x29, #0x90 0.00 : ffff8000101262fc: bl ffff8000101257a0 : if (!group) { 0.00 : ffff800010126300: cbz x0, ffff800010126d18 0.00 : ffff800010126304: adrp x5, ffff80001190a000 <__per_cpu_offset+0x6d8> : find_busiest_queue(): : rq = cpu_rq(i); 0.00 : ffff800010126308: adrp x22, ffff800011909000 : unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 0.00 : ffff80001012630c: mov x1, #0x1 // #1 : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff800010126310: add x28, x5, #0x2f0 : unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 0.00 : ffff800010126314: mov x21, x1 : rq = cpu_rq(i); 0.00 : ffff800010126318: add x22, x22, #0x928 0.00 : ffff80001012631c: add x23, x0, #0x20 : unsigned int busiest_nr = 0; 0.00 : ffff800010126320: mov w27, #0x0 // #0 : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff800010126324: mov w0, #0xffffffff // #-1 : unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 0.00 : ffff800010126328: mov x25, #0x0 // #0 0.00 : ffff80001012632c: mov x26, #0x0 // #0 : struct rq *busiest = NULL, *rq; 0.00 : ffff800010126330: mov x19, #0x0 // #0 0.00 : ffff800010126334: nop : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff800010126338: ldr x2, [x29, #200] 0.00 : ffff80001012633c: mov x1, x23 0.00 : ffff800010126340: bl ffff800010cb93f0 0.00 : ffff800010126344: ldr w1, [x28] 0.00 : ffff800010126348: cmp w0, w1 0.00 : ffff80001012634c: b.cs ffff800010126410 // b.hs, b.nlast : rq = cpu_rq(i); 0.00 : ffff800010126350: ldr x2, [x22, w0, sxtw #3] 0.00 : ffff800010126354: mov x1, x20 0.00 : ffff800010126358: add x1, x1, x2 : fbq_classify_rq(): : if (rq->nr_running > rq->nr_numa_running) 0.00 : ffff80001012635c: ldp w3, w4, [x1, #4] 0.00 : ffff800010126360: cmp w3, w4 0.00 : ffff800010126364: b.hi ffff800010126384 // b.pmore : if (rq->nr_running > rq->nr_preferred_running) 8.66 : ffff800010126368: ldr w6, [x1, #12] : find_busiest_queue(): : if (rt > env->fbq_type) 3.43 : ffff80001012636c: ldr w4, [x29, #224] : fbq_classify_rq(): : return remote; 0.00 : ffff800010126370: cmp w3, w6 0.00 : ffff800010126374: cset w3, ls // ls = plast 0.00 : ffff800010126378: add w3, w3, #0x1 : find_busiest_queue(): : if (rt > env->fbq_type) 0.00 : ffff80001012637c: cmp w4, w3 0.00 : ffff800010126380: b.cc ffff800010126338 // b.lo, b.ul, b.last : if (env->sd->flags & SD_ASYM_CPUCAPACITY && 3.55 : ffff800010126384: ldr x6, [x29, #144] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010126388: mov x3, x20 0.00 : ffff80001012638c: add x2, x2, x3 : find_busiest_queue(): : nr_running = rq->cfs.h_nr_running; 1.92 : ffff800010126390: ldr w8, [x1, #156] : if (env->sd->flags & SD_ASYM_CPUCAPACITY && 0.00 : ffff800010126394: ldr w7, [x6, #56] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010126398: ldr x4, [x2, #2480] : find_busiest_queue(): : if (env->sd->flags & SD_ASYM_CPUCAPACITY && 0.00 : ffff80001012639c: tbz w7, #6, ffff800010126668 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff8000101263a0: ldrsw x2, [x29, #164] 0.00 : ffff8000101263a4: ldr x2, [x22, x2, lsl #3] 0.00 : ffff8000101263a8: add x3, x2, x3 : find_busiest_queue(): : if (env->sd->flags & SD_ASYM_CPUCAPACITY && 0.00 : ffff8000101263ac: ldr x2, [x3, #2480] 0.00 : ffff8000101263b0: cmp x4, x2 0.00 : ffff8000101263b4: b.ls ffff800010126668 // b.plast : capacity_of(env->dst_cpu) < capacity && 0.00 : ffff8000101263b8: cmp w8, #0x1 0.00 : ffff8000101263bc: b.eq ffff800010126338 // b.none : switch (env->migration_type) { 0.00 : ffff8000101263c0: ldr w2, [x29, #228] 0.00 : ffff8000101263c4: cmp w2, #0x1 0.00 : ffff8000101263c8: b.eq ffff800010126884 // b.none 0.00 : ffff8000101263cc: cbz w2, ffff8000101269dc 0.00 : ffff8000101263d0: cmp w2, #0x2 0.00 : ffff8000101263d4: b.eq ffff8000101268c4 // b.none 0.00 : ffff8000101263d8: cmp w2, #0x3 0.00 : ffff8000101263dc: b.ne ffff800010126338 // b.any : if (rq->misfit_task_load > busiest_load) { 0.00 : ffff8000101263e0: ldr x2, [x1, #2512] : busiest = rq; 0.00 : ffff8000101263e4: cmp x25, x2 0.00 : ffff8000101263e8: csel x3, x2, x25, cc // cc = lo, ul, last : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff8000101263ec: ldr x2, [x29, #200] : busiest = rq; 0.00 : ffff8000101263f0: csel x19, x1, x19, cc // cc = lo, ul, last 0.00 : ffff8000101263f4: mov x25, x3 : for_each_cpu_and(i, sched_group_span(group), env->cpus) { 0.00 : ffff8000101263f8: mov x1, x23 0.00 : ffff8000101263fc: bl ffff800010cb93f0 0.00 : ffff800010126400: ldr w1, [x28] 0.00 : ffff800010126404: cmp w0, w1 0.00 : ffff800010126408: b.cc ffff800010126350 // b.lo, b.ul, b.last 0.00 : ffff80001012640c: nop : load_balance(): : schedstat_inc(sd->lb_nobusyg[idle]); : goto out_balanced; : } : : busiest = find_busiest_queue(&env, group); : if (!busiest) { 0.00 : ffff800010126410: cbz x19, ffff800010126d18 : schedstat_inc(sd->lb_nobusyq[idle]); : goto out_balanced; : } : : BUG_ON(busiest == env.dst_rq); 0.00 : ffff800010126414: ldr x0, [x29, #168] 0.00 : ffff800010126418: cmp x0, x19 0.00 : ffff80001012641c: b.eq ffff800010126b0c // b.none : : schedstat_add(sd->lb_imbalance[idle], env.imbalance); : : env.src_cpu = busiest->cpu; 0.00 : ffff800010126420: ldr w0, [x19, #2568] : env.src_rq = busiest; 0.15 : ffff800010126424: str x19, [x29, #152] : env.src_cpu = busiest->cpu; 0.00 : ffff800010126428: str w0, [x29, #160] : : ld_moved = 0; : if (busiest->nr_running > 1) { 0.00 : ffff80001012642c: ldr w0, [x19, #4] 0.00 : ffff800010126430: cmp w0, #0x1 0.00 : ffff800010126434: b.ls ffff800010126940 // b.plast : * Attempt to move tasks. If find_busiest_group has found : * an imbalance but busiest->nr_running <= 1, the group is : * still unbalanced. ld_moved simply stays zero, so it is : * correctly treated as an imbalance. : */ : env.flags |= LBF_ALL_PINNED; 0.00 : ffff800010126438: ldr w0, [x29, #208] : env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 0.00 : ffff80001012643c: adrp x1, ffff800010d06000 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010126440: adrp x27, ffff800011909000 : load_balance(): : ld_moved = 0; 0.00 : ffff800010126444: mov w26, #0x0 // #0 : env.flags |= LBF_ALL_PINNED; 0.00 : ffff800010126448: orr w0, w0, #0x1 0.00 : ffff80001012644c: str w0, [x29, #208] : env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 0.00 : ffff800010126450: ldr w0, [x1, #92] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010126454: add x2, x27, #0x928 : load_balance(): : env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 0.00 : ffff800010126458: ldr w1, [x19, #4] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff80001012645c: str x2, [x29, #136] : load_balance(): : env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 0.00 : ffff800010126460: cmp w0, w1 0.00 : ffff800010126464: csel w0, w0, w1, ls // ls = plast 0.00 : ffff800010126468: str w0, [x29, #220] 0.00 : ffff80001012646c: nop : rq_lock_irqsave(): : raw_spin_lock_irqsave(&rq->lock, rf->flags); 0.00 : ffff800010126470: mov x0, x19 0.00 : ffff800010126474: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff800010126478: mov x25, x0 : load_balance(): : : more_balance: : rq_lock_irqsave(busiest, &rf); : update_rq_clock(busiest); 0.00 : ffff80001012647c: mov x0, x19 0.00 : ffff800010126480: bl ffff800010114980 : detach_tasks(): : if (env->imbalance <= 0) 0.00 : ffff800010126484: ldr x0, [x29, #192] 0.00 : ffff800010126488: cmp x0, #0x0 0.00 : ffff80001012648c: b.le ffff80001012665c : struct list_head *tasks = &env->src_rq->cfs_tasks; 0.00 : ffff800010126490: ldr x28, [x29, #152] : int detached = 0; 0.00 : ffff800010126494: mov w23, #0x0 // #0 : struct list_head *tasks = &env->src_rq->cfs_tasks; 0.00 : ffff800010126498: add x22, x28, #0xa10 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001012649c: ldr x0, [x28, #2576] : detach_tasks(): : while (!list_empty(tasks)) { 0.00 : ffff8000101264a0: cmp x22, x0 0.00 : ffff8000101264a4: b.eq ffff8000101265bc // b.none : if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) 0.00 : ffff8000101264a8: ldr w0, [x29, #188] 0.00 : ffff8000101264ac: cmp w0, #0x1 0.00 : ffff8000101264b0: b.eq ffff8000101264c4 // b.none 0.00 : ffff8000101264b4: ldr x0, [x29, #152] 0.00 : ffff8000101264b8: ldr w0, [x0, #4] 0.00 : ffff8000101264bc: cmp w0, #0x1 0.00 : ffff8000101264c0: b.ls ffff8000101265bc // b.plast : p = list_last_entry(tasks, struct task_struct, se.group_node); 0.00 : ffff8000101264c4: add x27, x28, #0xa00 : env->loop++; 0.00 : ffff8000101264c8: ldr w0, [x29, #212] : if (env->loop > env->loop_max) 0.00 : ffff8000101264cc: ldr w1, [x29, #220] : env->loop++; 0.00 : ffff8000101264d0: add w0, w0, #0x1 : p = list_last_entry(tasks, struct task_struct, se.group_node); 1.32 : ffff8000101264d4: ldr x21, [x27, #24] : env->loop++; 0.00 : ffff8000101264d8: str w0, [x29, #212] : if (env->loop > env->loop_max) 0.00 : ffff8000101264dc: cmp w0, w1 : p = list_last_entry(tasks, struct task_struct, se.group_node); 0.00 : ffff8000101264e0: sub x24, x21, #0xf0 : if (env->loop > env->loop_max) 0.00 : ffff8000101264e4: b.hi ffff8000101265bc // b.pmore : if (env->loop > env->loop_break) { 0.00 : ffff8000101264e8: ldr w1, [x29, #216] 0.00 : ffff8000101264ec: cmp w0, w1 0.00 : ffff8000101264f0: b.hi ffff80001012686c // b.pmore : if (!can_migrate_task(p, env)) 0.00 : ffff8000101264f4: add x1, x29, #0x90 0.00 : ffff8000101264f8: mov x0, x24 0.00 : ffff8000101264fc: bl ffff800010122e68 0.00 : ffff800010126500: cbz w0, ffff8000101266a4 : switch (env->migration_type) { 0.00 : ffff800010126504: ldr w0, [x29, #228] 0.00 : ffff800010126508: cmp w0, #0x1 0.00 : ffff80001012650c: b.eq ffff8000101267b0 // b.none 0.00 : ffff800010126510: cbz w0, ffff800010126760 0.00 : ffff800010126514: cmp w0, #0x2 0.00 : ffff800010126518: b.eq ffff800010126750 // b.none 0.00 : ffff80001012651c: cmp w0, #0x3 0.00 : ffff800010126520: b.ne ffff800010126570 // b.any : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010126524: ldrsw x1, [x29, #160] 0.00 : ffff800010126528: mov x6, x20 : __read_once_size(): 0.00 : ffff80001012652c: ldr x8, [x21, #192] : capacity_of(): 0.00 : ffff800010126530: ldr x2, [x29, #136] : __read_once_size(): 0.00 : ffff800010126534: ldr x7, [x21, #200] : capacity_of(): 0.00 : ffff800010126538: ldr x1, [x2, x1, lsl #3] : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff80001012653c: lsr x0, x7, #32 0.00 : ffff800010126540: cmp w0, w7 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010126544: add x1, x1, x6 : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff800010126548: csel w0, w0, w7, hi // hi = pmore 0.00 : ffff80001012654c: orr w0, w0, #0x1 : task_util_est(): : return max(task_util(p), _task_util_est(p)); 0.00 : ffff800010126550: cmp x0, x8 : task_fits_capacity(): : return fits_capacity(uclamp_task_util(p), capacity); 0.00 : ffff800010126554: ldr x1, [x1, #2480] : task_util_est(): : return max(task_util(p), _task_util_est(p)); 0.00 : ffff800010126558: csel x0, x0, x8, cs // cs = hs, nlast : task_fits_capacity(): : return fits_capacity(uclamp_task_util(p), capacity); 0.00 : ffff80001012655c: add x0, x0, x0, lsl #2 0.00 : ffff800010126560: lsl x1, x1, #10 : detach_tasks(): : if (task_fits_capacity(p, capacity_of(env->src_cpu))) 0.00 : ffff800010126564: cmp x1, x0, lsl #8 0.00 : ffff800010126568: b.hi ffff8000101266a4 // b.pmore : env->imbalance = 0; 0.00 : ffff80001012656c: str xzr, [x29, #192] : detach_task(): : deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 0.00 : ffff800010126570: ldr x0, [x29, #152] 0.00 : ffff800010126574: mov w2, #0x8 // #8 0.00 : ffff800010126578: mov x1, x24 : detach_tasks(): : detached++; 0.00 : ffff80001012657c: add w23, w23, #0x1 : detach_task(): : deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 0.00 : ffff800010126580: bl ffff8000101150a8 : set_task_cpu(p, env->dst_cpu); 0.00 : ffff800010126584: ldr w1, [x29, #164] 0.00 : ffff800010126588: mov x0, x24 0.00 : ffff80001012658c: bl ffff800010116390 : list_add(): : * Insert a new entry after the specified head. : * This is good for implementing stacks. : */ : static inline void list_add(struct list_head *new, struct list_head *head) : { : __list_add(new, head, head->next); 0.00 : ffff800010126590: ldr x1, [x29, #232] : __list_add(): : new->prev = prev; 0.00 : ffff800010126594: add x0, x29, #0xe8 : next->prev = new; 0.00 : ffff800010126598: str x21, [x1, #8] : new->prev = prev; 0.00 : ffff80001012659c: stp x1, x0, [x21] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101265a0: str x21, [x29, #232] : detach_tasks(): : if (env->idle == CPU_NEWLY_IDLE) 0.00 : ffff8000101265a4: ldr w0, [x29, #188] 0.00 : ffff8000101265a8: cmp w0, #0x2 0.00 : ffff8000101265ac: b.eq ffff8000101265bc // b.none : if (env->imbalance <= 0) 0.00 : ffff8000101265b0: ldr x0, [x29, #192] 0.00 : ffff8000101265b4: cmp x0, #0x0 0.00 : ffff8000101265b8: b.gt ffff80001012649c : rq_unlock(): : raw_spin_unlock(&rq->lock); 0.00 : ffff8000101265bc: mov x0, x19 0.00 : ffff8000101265c0: bl ffff800010cd7c38 <_raw_spin_unlock> : load_balance(): : * See task_rq_lock() family for the details. : */ : : rq_unlock(busiest, &rf); : : if (cur_ld_moved) { 0.00 : ffff8000101265c4: cbz w23, ffff800010126624 : rq_lock(): : raw_spin_lock(&rq->lock); 0.00 : ffff8000101265c8: ldr x0, [x29, #168] : attach_tasks(): : while (!list_empty(tasks)) { 0.00 : ffff8000101265cc: add x22, x29, #0xe8 : rq_lock(): 0.00 : ffff8000101265d0: bl ffff800010cd80f0 <_raw_spin_lock> : attach_tasks(): : update_rq_clock(env->dst_rq); 0.00 : ffff8000101265d4: ldr x0, [x29, #168] 0.00 : ffff8000101265d8: bl ffff800010114980 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101265dc: ldr x0, [x29, #232] : attach_tasks(): : while (!list_empty(tasks)) { 0.00 : ffff8000101265e0: cmp x22, x0 0.00 : ffff8000101265e4: b.eq ffff800010126618 // b.none : p = list_first_entry(tasks, struct task_struct, se.group_node); 0.00 : ffff8000101265e8: ldr x0, [x29, #232] : attach_task(env->dst_rq, p); 0.00 : ffff8000101265ec: sub x1, x0, #0xf0 : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff8000101265f0: ldp x3, x2, [x0] : __list_del(): : next->prev = prev; 0.00 : ffff8000101265f4: str x2, [x3, #8] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101265f8: str x3, [x2] 0.00 : ffff8000101265fc: str x0, [x0] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010126600: str x0, [x0, #8] : attach_tasks(): 0.00 : ffff800010126604: ldr x0, [x29, #168] 0.00 : ffff800010126608: bl ffff80001011dcf0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012660c: ldr x0, [x29, #232] : attach_tasks(): : while (!list_empty(tasks)) { 0.00 : ffff800010126610: cmp x22, x0 0.00 : ffff800010126614: b.ne ffff8000101265e8 // b.any : rq_unlock(): : raw_spin_unlock(&rq->lock); 0.00 : ffff800010126618: ldr x0, [x29, #168] : load_balance(): : attach_tasks(&env); : ld_moved += cur_ld_moved; 0.00 : ffff80001012661c: add w26, w26, w23 : rq_unlock(): 0.00 : ffff800010126620: bl ffff800010cd7c38 <_raw_spin_unlock> : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff800010126624: msr daif, x25 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 1.60 : ffff800010126628: nop : load_balance(): : } : : local_irq_restore(rf.flags); : : if (env.flags & LBF_NEED_BREAK) { 0.00 : ffff80001012662c: ldr w0, [x29, #208] 0.00 : ffff800010126630: tbz w0, #1, ffff8000101266c4 : env.flags &= ~LBF_NEED_BREAK; 0.00 : ffff800010126634: and w0, w0, #0xfffffffd 0.00 : ffff800010126638: str w0, [x29, #208] : rq_lock_irqsave(): : raw_spin_lock_irqsave(&rq->lock, rf->flags); 0.00 : ffff80001012663c: mov x0, x19 0.00 : ffff800010126640: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff800010126644: mov x25, x0 : load_balance(): : update_rq_clock(busiest); 0.00 : ffff800010126648: mov x0, x19 0.00 : ffff80001012664c: bl ffff800010114980 : detach_tasks(): : if (env->imbalance <= 0) 0.00 : ffff800010126650: ldr x0, [x29, #192] 0.00 : ffff800010126654: cmp x0, #0x0 0.00 : ffff800010126658: b.gt ffff800010126490 : rq_unlock(): : raw_spin_unlock(&rq->lock); 0.00 : ffff80001012665c: mov x0, x19 0.00 : ffff800010126660: bl ffff800010cd7c38 <_raw_spin_unlock> 0.00 : ffff800010126664: b ffff800010126624 : find_busiest_queue(): : switch (env->migration_type) { 0.00 : ffff800010126668: ldr w2, [x29, #228] 0.00 : ffff80001012666c: cmp w2, #0x1 0.00 : ffff800010126670: b.eq ffff800010126884 // b.none 0.00 : ffff800010126674: cbnz w2, ffff8000101263d0 : if (nr_running == 1 && load > env->imbalance && 0.00 : ffff800010126678: cmp w8, #0x1 : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff80001012667c: ldr x2, [x1, #288] : find_busiest_queue(): : if (nr_running == 1 && load > env->imbalance && 0.00 : ffff800010126680: b.eq ffff8000101269ac // b.none : if (load * busiest_capacity > busiest_load * capacity) { 0.00 : ffff800010126684: mul x3, x4, x25 0.00 : ffff800010126688: mul x6, x21, x2 : busiest = rq; 0.00 : ffff80001012668c: cmp x6, x3 0.00 : ffff800010126690: csel x4, x4, x21, hi // hi = pmore 0.00 : ffff800010126694: csel x25, x2, x25, hi // hi = pmore 0.00 : ffff800010126698: mov x21, x4 0.00 : ffff80001012669c: csel x19, x1, x19, hi // hi = pmore 0.00 : ffff8000101266a0: b ffff800010126338 : __list_del_entry(): : __list_del(entry->prev, entry->next); 0.00 : ffff8000101266a4: ldp x1, x0, [x21] : __list_del(): : next->prev = prev; 1.67 : ffff8000101266a8: str x0, [x1, #8] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101266ac: str x1, [x0] : list_add(): : __list_add(new, head, head->next); 0.00 : ffff8000101266b0: ldr x0, [x27, #16] : __list_add(): : next->prev = new; 1.59 : ffff8000101266b4: str x21, [x0, #8] : new->prev = prev; 0.00 : ffff8000101266b8: stp x0, x22, [x21] : __write_once_size(): 0.00 : ffff8000101266bc: str x21, [x28, #2576] 0.00 : ffff8000101266c0: b ffff80001012649c : load_balance(): : * given_cpu) causing exceess load to be moved to given_cpu. : * This however should not happen so much in practice and : * moreover subsequent load balance cycles should correct the : * excess load moved. : */ : if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 1.85 : ffff8000101266c4: tbz w0, #2, ffff8000101267e8 0.00 : ffff8000101266c8: ldr x1, [x29, #192] 0.00 : ffff8000101266cc: cmp x1, #0x0 0.00 : ffff8000101266d0: b.le ffff8000101267e8 : : /* Prevent to re-select dst_cpu via env's CPUs */ : __cpumask_clear_cpu(env.dst_cpu, env.cpus); 0.00 : ffff8000101266d4: ldr w1, [x29, #164] : __clear_bit(): : unsigned long mask = BIT_MASK(nr); 0.00 : ffff8000101266d8: mov x2, #0x1 // #1 : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff8000101266dc: ldr x5, [x29, #200] : load_balance(): : : env.dst_rq = cpu_rq(env.new_dst_cpu); 0.00 : ffff8000101266e0: mov x4, x20 : __clear_bit(): 0.00 : ffff8000101266e4: cmp w1, #0x0 0.00 : ffff8000101266e8: add w0, w1, #0x3f 0.00 : ffff8000101266ec: csel w0, w0, w1, lt // lt = tstop : unsigned long mask = BIT_MASK(nr); 0.00 : ffff8000101266f0: negs w3, w1 0.00 : ffff8000101266f4: and w3, w3, #0x3f 0.00 : ffff8000101266f8: and w1, w1, #0x3f : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff8000101266fc: asr w0, w0, #6 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010126700: csneg w1, w1, w3, mi // mi = first : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff800010126704: sxtw x0, w0 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010126708: lsl x1, x2, x1 : load_balance(): : env.dst_cpu = env.new_dst_cpu; : env.flags &= ~LBF_DST_PINNED; : env.loop = 0; 0.00 : ffff80001012670c: mov x2, #0x2000000000 // #137438953472 : __clear_bit(): : *p &= ~mask; 0.00 : ffff800010126710: ldr x3, [x5, x0, lsl #3] 0.00 : ffff800010126714: bic x1, x3, x1 0.00 : ffff800010126718: str x1, [x5, x0, lsl #3] : load_balance(): 0.00 : ffff80001012671c: stur x2, [x29, #212] : env.flags &= ~LBF_DST_PINNED; 0.00 : ffff800010126720: ldr w0, [x29, #208] : env.dst_rq = cpu_rq(env.new_dst_cpu); 0.00 : ffff800010126724: ldr w1, [x29, #184] : env.flags &= ~LBF_DST_PINNED; 0.00 : ffff800010126728: and w0, w0, #0xfffffffb 0.00 : ffff80001012672c: str w0, [x29, #208] : env.dst_rq = cpu_rq(env.new_dst_cpu); 0.00 : ffff800010126730: ldr x0, [x29, #136] : env.dst_cpu = env.new_dst_cpu; 0.00 : ffff800010126734: str w1, [x29, #164] : env.dst_rq = cpu_rq(env.new_dst_cpu); 0.00 : ffff800010126738: ldr x0, [x0, w1, sxtw #3] 0.00 : ffff80001012673c: add x0, x0, x4 0.00 : ffff800010126740: str x0, [x29, #168] : : /* : * Go back to "more_balance" rather than "redo" since we : * need to continue with same src_cpu. : */ : goto more_balance; 0.00 : ffff800010126744: b ffff800010126470 : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff800010126748: dsb sy 4.22 : ffff80001012674c: b ffff80001012662c : detach_tasks(): : env->imbalance--; 0.00 : ffff800010126750: ldr x0, [x29, #192] 0.00 : ffff800010126754: sub x0, x0, #0x1 0.00 : ffff800010126758: str x0, [x29, #192] 0.00 : ffff80001012675c: b ffff800010126570 : task_cfs_rq(): : return p->se.cfs_rq; 0.00 : ffff800010126760: ldr x6, [x24, #320] : task_h_load(): : update_cfs_rq_h_load(cfs_rq); 0.00 : ffff800010126764: mov x0, x6 0.00 : ffff800010126768: bl ffff80001011c898 : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.00 : ffff80001012676c: ldr x7, [x6, #280] 0.00 : ffff800010126770: ldr x0, [x24, #416] 0.00 : ffff800010126774: ldr x6, [x6, #160] : detach_tasks(): : if (load/2 > env->imbalance && 0.00 : ffff800010126778: ldr x1, [x29, #192] : task_h_load(): : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.00 : ffff80001012677c: add x6, x6, #0x1 0.00 : ffff800010126780: mul x0, x0, x7 : div64_u64(): : * : * Return: dividend / divisor : */ : static inline u64 div64_u64(u64 dividend, u64 divisor) : { : return dividend / divisor; 0.00 : ffff800010126784: udiv x0, x0, x6 : detach_tasks(): : if (load/2 > env->imbalance && 0.00 : ffff800010126788: cmp x1, x0, lsr #1 0.00 : ffff80001012678c: b.cs ffff8000101267a4 // b.hs, b.nlast : env->sd->nr_balance_failed <= env->sd->cache_nice_tries) 0.00 : ffff800010126790: ldr x6, [x29, #144] : if (load/2 > env->imbalance && 0.00 : ffff800010126794: ldr w7, [x6, #48] 0.00 : ffff800010126798: ldr w6, [x6, #76] 0.00 : ffff80001012679c: cmp w6, w7 0.00 : ffff8000101267a0: b.ls ffff8000101266a4 // b.plast : env->imbalance -= load; 0.00 : ffff8000101267a4: sub x0, x1, x0 0.00 : ffff8000101267a8: str x0, [x29, #192] 0.00 : ffff8000101267ac: b ffff800010126570 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101267b0: ldr x7, [x21, #192] 0.00 : ffff8000101267b4: ldr x1, [x21, #200] : detach_tasks(): : if (util > env->imbalance) 0.00 : ffff8000101267b8: ldr x6, [x29, #192] : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff8000101267bc: lsr x0, x1, #32 0.00 : ffff8000101267c0: cmp w0, w1 0.00 : ffff8000101267c4: csel w0, w0, w1, hi // hi = pmore 0.00 : ffff8000101267c8: orr w0, w0, #0x1 : task_util_est(): : return max(task_util(p), _task_util_est(p)); 0.00 : ffff8000101267cc: cmp x0, x7 0.00 : ffff8000101267d0: csel x0, x0, x7, cs // cs = hs, nlast : detach_tasks(): : if (util > env->imbalance) 0.00 : ffff8000101267d4: cmp x0, x6 0.00 : ffff8000101267d8: b.hi ffff8000101266a4 // b.pmore : env->imbalance -= util; 0.00 : ffff8000101267dc: sub x0, x6, x0 0.00 : ffff8000101267e0: str x0, [x29, #192] 0.00 : ffff8000101267e4: b ffff800010126570 : load_balance(): : } : : /* : * We failed to reach balance because of affinity. : */ : if (sd_parent) { 0.00 : ffff8000101267e8: ldr x2, [x29, #120] 0.00 : ffff8000101267ec: cbz x2, ffff800010126814 : int *group_imbalance = &sd_parent->groups->sgc->imbalance; : : if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 0.00 : ffff8000101267f0: tbz w0, #3, ffff800010126814 0.00 : ffff8000101267f4: ldr x1, [x29, #192] 0.00 : ffff8000101267f8: cmp x1, #0x0 0.00 : ffff8000101267fc: b.le ffff800010126814 : int *group_imbalance = &sd_parent->groups->sgc->imbalance; 0.00 : ffff800010126800: ldr x0, [x2, #16] : *group_imbalance = 1; 0.00 : ffff800010126804: mov w1, #0x1 // #1 : int *group_imbalance = &sd_parent->groups->sgc->imbalance; 0.00 : ffff800010126808: ldr x0, [x0, #16] : *group_imbalance = 1; 3.41 : ffff80001012680c: str w1, [x0, #40] 0.00 : ffff800010126810: ldr w0, [x29, #208] : } : : /* All tasks on this runqueue were pinned by CPU affinity */ : if (unlikely(env.flags & LBF_ALL_PINNED)) { 0.00 : ffff800010126814: tbnz w0, #0, ffff800010126b10 : } : goto out_all_pinned; : } : } : : if (!ld_moved) { 0.00 : ffff800010126818: cbz w26, ffff800010126940 : : /* We've kicked active balancing, force task migration. */ : sd->nr_balance_failed = sd->cache_nice_tries+1; : } : } else : sd->nr_balance_failed = 0; 0.00 : ffff80001012681c: ldr x0, [x29, #128] 0.00 : ffff800010126820: str wzr, [x0, #76] : : if (likely(!active_balance) || voluntary_active_balance(&env)) { : /* We were unbalanced, so reset the balancing interval */ : sd->balance_interval = sd->min_interval; 0.00 : ffff800010126824: ldr x1, [x29, #128] 0.00 : ffff800010126828: ldr x0, [x1, #24] : * case may not be covered by the all_pinned logic if there : * is only 1 task on the busy runqueue (because we don't call : * detach_tasks). : */ : if (sd->balance_interval < sd->max_interval) : sd->balance_interval *= 2; 0.00 : ffff80001012682c: str w0, [x1, #72] 0.00 : ffff800010126830: ldp x19, x20, [x29, #16] 0.00 : ffff800010126834: ldp x22, x23, [x29, #40] 0.00 : ffff800010126838: ldp x24, x25, [x29, #56] 0.00 : ffff80001012683c: ldp x27, x28, [x29, #80] : sd->balance_interval < MAX_PINNED_INTERVAL) || : sd->balance_interval < sd->max_interval) : sd->balance_interval *= 2; : out: : return ld_moved; : } 0.00 : ffff800010126840: adrp x0, ffff800011909000 0.00 : ffff800010126844: add x1, x0, #0x908 0.00 : ffff800010126848: mov w0, w26 1.78 : ffff80001012684c: ldr x2, [x29, #248] 0.00 : ffff800010126850: ldr x1, [x1] 0.00 : ffff800010126854: eor x1, x2, x1 0.00 : ffff800010126858: cbnz x1, ffff800010126d58 0.00 : ffff80001012685c: ldr x21, [sp, #32] 0.00 : ffff800010126860: ldr x26, [sp, #72] 0.00 : ffff800010126864: ldp x29, x30, [sp], #256 0.00 : ffff800010126868: ret : detach_tasks(): : env->flags |= LBF_NEED_BREAK; 0.00 : ffff80001012686c: ldr w0, [x29, #208] : env->loop_break += sched_nr_migrate_break; 0.00 : ffff800010126870: add w1, w1, #0x20 0.00 : ffff800010126874: str w1, [x29, #216] : env->flags |= LBF_NEED_BREAK; 0.00 : ffff800010126878: orr w0, w0, #0x2 0.00 : ffff80001012687c: str w0, [x29, #208] 0.00 : ffff800010126880: b ffff8000101265bc : cpu_util(): : cfs_rq = &cpu_rq(cpu)->cfs; 0.00 : ffff800010126884: ldrsw x3, [x1, #2568] 0.00 : ffff800010126888: mov x2, x20 1.45 : ffff80001012688c: ldr x3, [x22, x3, lsl #3] 0.00 : ffff800010126890: add x2, x2, x3 : __read_once_size(): 0.00 : ffff800010126894: ldr x3, [x2, #304] 0.00 : ffff800010126898: ldr w4, [x2, #312] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff80001012689c: ldr x2, [x2, #2488] : util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); 0.00 : ffff8000101268a0: cmp w4, w3 : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff8000101268a4: csel w3, w3, w4, ls // ls = plast 0.00 : ffff8000101268a8: cmp x2, x3 0.00 : ffff8000101268ac: csel x2, x2, x3, ls // ls = plast : find_busiest_queue(): : busiest = rq; 0.00 : ffff8000101268b0: cmp x2, x26 0.00 : ffff8000101268b4: csel x3, x2, x26, hi // hi = pmore 0.00 : ffff8000101268b8: csel x19, x1, x19, hi // hi = pmore 0.00 : ffff8000101268bc: mov x26, x3 0.00 : ffff8000101268c0: b ffff800010126338 : busiest = rq; 0.00 : ffff8000101268c4: cmp w8, w27 0.00 : ffff8000101268c8: csel w2, w8, w27, hi // hi = pmore 0.00 : ffff8000101268cc: csel x19, x1, x19, hi // hi = pmore 0.00 : ffff8000101268d0: mov w27, w2 0.00 : ffff8000101268d4: b ffff800010126338 0.00 : ffff8000101268d8: ldp x19, x20, [x29, #16] 1.70 : ffff8000101268dc: ldp x22, x23, [x29, #40] 0.00 : ffff8000101268e0: ldp x24, x25, [x29, #56] 0.00 : ffff8000101268e4: ldp x27, x28, [x29, #80] : load_balance(): : *continue_balancing = 0; 0.00 : ffff8000101268e8: ldr x0, [x29, #112] 0.00 : ffff8000101268ec: str wzr, [x0] : if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 0.00 : ffff8000101268f0: ldr x1, [x29, #120] 0.00 : ffff8000101268f4: cbz x1, ffff800010126900 0.00 : ffff8000101268f8: ldr w0, [x29, #208] 0.00 : ffff8000101268fc: tbz w0, #0, ffff800010126cd4 : sd->nr_balance_failed = 0; 8.60 : ffff800010126900: ldr x0, [x29, #128] 0.00 : ffff800010126904: str wzr, [x0, #76] : if (env.idle == CPU_NEWLY_IDLE) 0.00 : ffff800010126908: ldr w0, [x29, #188] 0.00 : ffff80001012690c: cmp w0, #0x2 0.00 : ffff800010126910: b.eq ffff800010126afc // b.none : if ((env.flags & LBF_ALL_PINNED && 0.00 : ffff800010126914: ldr w1, [x29, #208] 0.00 : ffff800010126918: ldr x0, [x29, #128] 0.00 : ffff80001012691c: ldr w0, [x0, #72] 0.00 : ffff800010126920: tbz w1, #0, ffff800010126aec 0.00 : ffff800010126924: cmp w0, #0x1ff 0.00 : ffff800010126928: b.hi ffff800010126aec // b.pmore : sd->balance_interval *= 2; 0.00 : ffff80001012692c: ldr x1, [x29, #128] 0.00 : ffff800010126930: lsl w0, w0, #1 : ld_moved = 0; 0.00 : ffff800010126934: mov w26, #0x0 // #0 : sd->balance_interval *= 2; 1.70 : ffff800010126938: str w0, [x1, #72] : return ld_moved; 0.00 : ffff80001012693c: b ffff800010126840 : if (idle != CPU_NEWLY_IDLE) 0.00 : ffff800010126940: ldr w0, [x29, #108] 0.00 : ffff800010126944: cmp w0, #0x2 0.00 : ffff800010126948: b.eq ffff80001012695c // b.none : sd->nr_balance_failed++; 0.00 : ffff80001012694c: ldr x1, [x29, #128] 0.00 : ffff800010126950: ldr w0, [x1, #76] 0.00 : ffff800010126954: add w0, w0, #0x1 0.00 : ffff800010126958: str w0, [x1, #76] : asym_active_balance(): : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 0.00 : ffff80001012695c: ldr w0, [x29, #188] : need_active_balance(): : struct sched_domain *sd = env->sd; 0.00 : ffff800010126960: ldr x20, [x29, #144] : asym_active_balance(): : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 0.00 : ffff800010126964: cmp w0, #0x1 0.00 : ffff800010126968: b.eq ffff800010126984 // b.none 0.00 : ffff80001012696c: ldr w0, [x20, #56] 0.00 : ffff800010126970: tbnz w0, #11, ffff800010126a00 : voluntary_active_balance(): : (env->src_rq->cfs.h_nr_running == 1)) { 0.00 : ffff800010126974: ldr x0, [x29, #152] : if ((env->idle != CPU_NOT_IDLE) && 0.00 : ffff800010126978: ldr w1, [x0, #156] 0.00 : ffff80001012697c: cmp w1, #0x1 0.00 : ffff800010126980: b.eq ffff800010126a2c // b.none : if (env->migration_type == migrate_misfit) 0.00 : ffff800010126984: ldr w0, [x29, #228] 0.00 : ffff800010126988: cmp w0, #0x3 0.00 : ffff80001012698c: b.eq ffff800010126a8c // b.none : need_active_balance(): : return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); 0.00 : ffff800010126990: ldr w0, [x20, #48] : load_balance(): : if (need_active_balance(&env)) { 0.00 : ffff800010126994: ldr w1, [x20, #76] : need_active_balance(): : return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); 0.00 : ffff800010126998: add w0, w0, #0x2 : load_balance(): : if (need_active_balance(&env)) { 0.00 : ffff80001012699c: cmp w1, w0 0.00 : ffff8000101269a0: b.hi ffff800010126a8c // b.pmore : sd->nr_balance_failed = sd->cache_nice_tries+1; 0.00 : ffff8000101269a4: mov w26, #0x0 // #0 0.00 : ffff8000101269a8: b ffff800010126824 : find_busiest_queue(): : if (nr_running == 1 && load > env->imbalance && 0.00 : ffff8000101269ac: ldr x3, [x29, #192] 0.00 : ffff8000101269b0: cmp x2, x3 0.00 : ffff8000101269b4: b.ls ffff800010126684 // b.plast : check_cpu_capacity(): : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff8000101269b8: ldr w3, [x6, #44] 0.00 : ffff8000101269bc: ldr x6, [x1, #2480] : (rq->cpu_capacity_orig * 100)); 0.00 : ffff8000101269c0: ldr x7, [x1, #2488] : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff8000101269c4: mul x3, x3, x6 : (rq->cpu_capacity_orig * 100)); 0.00 : ffff8000101269c8: add x6, x7, x7, lsl #1 0.00 : ffff8000101269cc: add x6, x7, x6, lsl #3 : find_busiest_queue(): : if (nr_running == 1 && load > env->imbalance && 0.00 : ffff8000101269d0: cmp x3, x6, lsl #2 0.00 : ffff8000101269d4: b.cs ffff800010126338 // b.hs, b.nlast 0.00 : ffff8000101269d8: b ffff800010126684 : cpu_load(): : return cfs_rq_load_avg(&rq->cfs); 0.00 : ffff8000101269dc: ldr x2, [x1, #288] : find_busiest_queue(): : if (load * busiest_capacity > busiest_load * capacity) { 0.00 : ffff8000101269e0: mul x3, x4, x25 0.00 : ffff8000101269e4: mul x6, x21, x2 : busiest = rq; 0.00 : ffff8000101269e8: cmp x6, x3 0.00 : ffff8000101269ec: csel x4, x4, x21, hi // hi = pmore 0.00 : ffff8000101269f0: csel x25, x2, x25, hi // hi = pmore 0.00 : ffff8000101269f4: mov x21, x4 0.00 : ffff8000101269f8: csel x19, x1, x19, hi // hi = pmore 0.00 : ffff8000101269fc: b ffff800010126338 : sched_asym_prefer(): : return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b); 0.00 : ffff800010126a00: ldp w22, w0, [x29, #160] 0.00 : ffff800010126a04: bl ffff800010123c30 0.00 : ffff800010126a08: mov w21, w0 0.00 : ffff800010126a0c: mov w0, w22 0.00 : ffff800010126a10: bl ffff800010123c30 : asym_active_balance(): : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 0.00 : ffff800010126a14: cmp w21, w0 0.00 : ffff800010126a18: b.gt ffff800010126a8c : voluntary_active_balance(): : if ((env->idle != CPU_NOT_IDLE) && 0.00 : ffff800010126a1c: ldr w0, [x29, #188] 0.00 : ffff800010126a20: cmp w0, #0x1 0.00 : ffff800010126a24: b.ne ffff800010126974 // b.any 0.00 : ffff800010126a28: b ffff800010126984 : check_cpu_capacity(): : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010126a2c: ldr x2, [x0, #2480] : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010126a30: ldr x4, [x0, #2488] : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010126a34: ldr w1, [x20, #44] : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010126a38: add x3, x4, x4, lsl #1 : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010126a3c: mul x0, x1, x2 : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010126a40: add x2, x4, x3, lsl #3 : voluntary_active_balance(): : if ((check_cpu_capacity(env->src_rq, sd)) && 0.00 : ffff800010126a44: cmp x0, x2, lsl #2 0.00 : ffff800010126a48: b.cs ffff800010126984 // b.hs, b.nlast : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010126a4c: ldpsw x3, x4, [x29, #160] 0.00 : ffff800010126a50: adrp x2, ffff800011909000 0.00 : ffff800010126a54: add x2, x2, #0x928 0.00 : ffff800010126a58: adrp x0, ffff800011528000 0.00 : ffff800010126a5c: add x0, x0, #0xe80 0.00 : ffff800010126a60: ldr x3, [x2, x3, lsl #3] 0.00 : ffff800010126a64: ldr x2, [x2, x4, lsl #3] 0.00 : ffff800010126a68: add x3, x3, x0 0.00 : ffff800010126a6c: add x0, x2, x0 : voluntary_active_balance(): : (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 0.00 : ffff800010126a70: ldr x3, [x3, #2480] 0.00 : ffff800010126a74: ldr x2, [x0, #2480] 0.00 : ffff800010126a78: add x0, x2, x2, lsl #1 0.00 : ffff800010126a7c: mul x1, x1, x3 0.00 : ffff800010126a80: add x0, x2, x0, lsl #3 : if ((check_cpu_capacity(env->src_rq, sd)) && 0.00 : ffff800010126a84: cmp x1, x0, lsl #2 0.00 : ffff800010126a88: b.cs ffff800010126984 // b.hs, b.nlast : load_balance(): : raw_spin_lock_irqsave(&busiest->lock, flags); 0.00 : ffff800010126a8c: mov x0, x19 0.00 : ffff800010126a90: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : test_bit(): : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010126a94: ldr w1, [x29, #104] : load_balance(): : if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { 0.00 : ffff800010126a98: ldr x3, [x19, #2352] : test_bit(): 0.00 : ffff800010126a9c: add w2, w1, #0x3f 0.00 : ffff800010126aa0: cmp w1, #0x0 0.00 : ffff800010126aa4: csel w2, w2, w1, lt // lt = tstop 0.00 : ffff800010126aa8: ldr x3, [x3, #736] 0.00 : ffff800010126aac: asr w2, w2, #6 0.00 : ffff800010126ab0: sxtw x2, w2 0.00 : ffff800010126ab4: ldr x2, [x3, x2, lsl #3] 0.00 : ffff800010126ab8: lsr x2, x2, x1 : load_balance(): 0.00 : ffff800010126abc: tbz w2, #0, ffff800010126ca8 : if (!busiest->active_balance) { 0.00 : ffff800010126ac0: ldr w26, [x19, #2520] 0.00 : ffff800010126ac4: cbz w26, ffff800010126bb0 : raw_spin_unlock_irqrestore(&busiest->lock, flags); 0.00 : ffff800010126ac8: mov x1, x0 0.00 : ffff800010126acc: mov x0, x19 0.00 : ffff800010126ad0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : sd->nr_balance_failed = sd->cache_nice_tries+1; 0.00 : ffff800010126ad4: mov w26, #0x0 // #0 0.00 : ffff800010126ad8: ldr x1, [x29, #128] 0.00 : ffff800010126adc: ldr w0, [x1, #48] 0.00 : ffff800010126ae0: add w0, w0, #0x1 0.00 : ffff800010126ae4: str w0, [x1, #76] 0.00 : ffff800010126ae8: b ffff800010126824 : sd->balance_interval < MAX_PINNED_INTERVAL) || 0.00 : ffff800010126aec: ldr x1, [x29, #128] 0.00 : ffff800010126af0: ldr x1, [x1, #32] 0.00 : ffff800010126af4: cmp x1, w0, uxtw 0.00 : ffff800010126af8: b.hi ffff80001012692c // b.pmore : sd->nr_balance_failed = sd->cache_nice_tries+1; 1.52 : ffff800010126afc: mov w26, #0x0 // #0 0.00 : ffff800010126b00: b ffff800010126840 0.00 : ffff800010126b04: ldr x2, [x29, #200] 0.00 : ffff800010126b08: b ffff8000101262bc : BUG_ON(busiest == env.dst_rq); 0.00 : ffff800010126b0c: brk #0x800 : __cpumask_clear_cpu(cpu_of(busiest), cpus); 0.00 : ffff800010126b10: ldr w3, [x19, #2568] : __clear_bit(): : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010126b14: mov x4, #0x1 // #1 : bitmap_subset(): : const unsigned long *src2, unsigned int nbits) : { : if (small_const_nbits(nbits)) : return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits)); : else : return __bitmap_subset(src1, src2, nbits); 0.00 : ffff800010126b18: ldr x6, [x29, #96] 0.00 : ffff800010126b1c: mov w2, #0x100 // #256 : __clear_bit(): : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff800010126b20: cmp w3, #0x0 0.00 : ffff800010126b24: add w1, w3, #0x3f 0.00 : ffff800010126b28: csel w1, w1, w3, lt // lt = tstop : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010126b2c: negs w5, w3 0.00 : ffff800010126b30: and w5, w5, #0x3f 0.00 : ffff800010126b34: and w3, w3, #0x3f : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff800010126b38: asr w1, w1, #6 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010126b3c: csneg w3, w3, w5, mi // mi = first : bitmap_subset(): 0.00 : ffff800010126b40: mov x0, x6 : __clear_bit(): : unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 0.00 : ffff800010126b44: sxtw x1, w1 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff800010126b48: lsl x3, x4, x3 : *p &= ~mask; 0.00 : ffff800010126b4c: ldr x4, [x6, x1, lsl #3] 0.00 : ffff800010126b50: bic x3, x4, x3 0.00 : ffff800010126b54: str x3, [x6, x1, lsl #3] : bitmap_subset(): 0.00 : ffff800010126b58: ldr x1, [x29, #176] 0.00 : ffff800010126b5c: bl ffff8000104852f0 <__bitmap_subset> : load_balance(): : if (!cpumask_subset(cpus, env.dst_grpmask)) { 0.00 : ffff800010126b60: cbnz w0, ffff800010126d44 : should_we_balance(): : if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 0.00 : ffff800010126b64: ldr w1, [x29, #164] : load_balance(): : env.loop = 0; 0.00 : ffff800010126b68: mov x0, #0x2000000000 // #137438953472 0.00 : ffff800010126b6c: stur x0, [x29, #212] : test_bit(): : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010126b70: cmp w1, #0x0 0.00 : ffff800010126b74: add w0, w1, #0x3f 0.00 : ffff800010126b78: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff800010126b7c: ldr x2, [x29, #200] 0.00 : ffff800010126b80: asr w0, w0, #6 0.00 : ffff800010126b84: sxtw x0, w0 0.00 : ffff800010126b88: ldr x0, [x2, x0, lsl #3] 0.00 : ffff800010126b8c: lsr x1, x0, x1 : should_we_balance(): : if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 0.00 : ffff800010126b90: tbz w1, #0, ffff8000101268d8 : struct sched_group *sg = env->sd->groups; 0.00 : ffff800010126b94: ldr x0, [x29, #144] 0.00 : ffff800010126b98: ldr x19, [x0, #16] 0.00 : ffff800010126b9c: b ffff8000101262a4 : balance_cpu = group_balance_cpu(sg); 0.00 : ffff800010126ba0: mov x0, x19 0.00 : ffff800010126ba4: bl ffff800010133098 0.00 : ffff800010126ba8: mov w23, w0 0.00 : ffff800010126bac: b ffff8000101262ec : load_balance(): : busiest->active_balance = 1; 0.00 : ffff800010126bb0: mov w1, #0x1 // #1 0.00 : ffff800010126bb4: str w1, [x19, #2520] : busiest->push_cpu = this_cpu; 0.00 : ffff800010126bb8: ldr w1, [x29, #104] 0.00 : ffff800010126bbc: str w1, [x19, #2524] : raw_spin_unlock_irqrestore(&busiest->lock, flags); 0.00 : ffff800010126bc0: mov x1, x0 0.00 : ffff800010126bc4: mov x0, x19 0.00 : ffff800010126bc8: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : stop_one_cpu_nowait(cpu_of(busiest), 0.00 : ffff800010126bcc: ldr w0, [x19, #2568] 0.00 : ffff800010126bd0: mov x2, x19 0.00 : ffff800010126bd4: adrp x1, ffff800010123000 0.00 : ffff800010126bd8: add x3, x19, #0x9e0 0.00 : ffff800010126bdc: add x1, x1, #0x1e8 0.00 : ffff800010126be0: bl ffff8000101a2cc8 : sd->nr_balance_failed = sd->cache_nice_tries+1; 0.00 : ffff800010126be4: ldr x2, [x29, #128] : asym_active_balance(): : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 0.00 : ffff800010126be8: ldr w1, [x29, #188] : load_balance(): : sd->nr_balance_failed = sd->cache_nice_tries+1; 0.00 : ffff800010126bec: ldr w0, [x2, #48] : asym_active_balance(): : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 0.00 : ffff800010126bf0: cmp w1, #0x1 : load_balance(): : sd->nr_balance_failed = sd->cache_nice_tries+1; 0.00 : ffff800010126bf4: add w0, w0, #0x1 0.00 : ffff800010126bf8: str w0, [x2, #76] : asym_active_balance(): : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 0.00 : ffff800010126bfc: b.eq ffff800010126c7c // b.none : voluntary_active_balance(): : struct sched_domain *sd = env->sd; 0.00 : ffff800010126c00: ldr x19, [x29, #144] : asym_active_balance(): : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 0.00 : ffff800010126c04: ldr w0, [x19, #56] 0.00 : ffff800010126c08: tbnz w0, #11, ffff800010126cec : voluntary_active_balance(): : (env->src_rq->cfs.h_nr_running == 1)) { 0.00 : ffff800010126c0c: ldr x2, [x29, #152] : if ((env->idle != CPU_NOT_IDLE) && 0.00 : ffff800010126c10: ldr w0, [x2, #156] 0.00 : ffff800010126c14: cmp w0, #0x1 0.00 : ffff800010126c18: b.ne ffff800010126c7c // b.any : check_cpu_capacity(): : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010126c1c: ldr x0, [x2, #2480] : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010126c20: ldr x3, [x2, #2488] : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010126c24: ldr w1, [x19, #44] : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010126c28: add x2, x3, x3, lsl #1 : return ((rq->cpu_capacity * sd->imbalance_pct) < 0.00 : ffff800010126c2c: mul x0, x1, x0 : (rq->cpu_capacity_orig * 100)); 0.00 : ffff800010126c30: add x2, x3, x2, lsl #3 : voluntary_active_balance(): : if ((check_cpu_capacity(env->src_rq, sd)) && 0.00 : ffff800010126c34: cmp x0, x2, lsl #2 0.00 : ffff800010126c38: b.cs ffff800010126c7c // b.hs, b.nlast : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010126c3c: ldpsw x3, x4, [x29, #160] 0.00 : ffff800010126c40: adrp x2, ffff800011909000 0.00 : ffff800010126c44: add x2, x2, #0x928 0.00 : ffff800010126c48: adrp x0, ffff800011528000 0.00 : ffff800010126c4c: add x0, x0, #0xe80 0.00 : ffff800010126c50: ldr x3, [x2, x3, lsl #3] 0.00 : ffff800010126c54: ldr x2, [x2, x4, lsl #3] 0.00 : ffff800010126c58: add x3, x3, x0 0.00 : ffff800010126c5c: add x0, x2, x0 : voluntary_active_balance(): : (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 0.00 : ffff800010126c60: ldr x3, [x3, #2480] 0.00 : ffff800010126c64: ldr x2, [x0, #2480] 0.00 : ffff800010126c68: add x0, x2, x2, lsl #1 0.00 : ffff800010126c6c: mul x1, x1, x3 0.00 : ffff800010126c70: add x0, x2, x0, lsl #3 : if ((check_cpu_capacity(env->src_rq, sd)) && 0.00 : ffff800010126c74: cmp x1, x0, lsl #2 0.00 : ffff800010126c78: b.cc ffff8000101269a4 // b.lo, b.ul, b.last : if (env->migration_type == migrate_misfit) 0.00 : ffff800010126c7c: ldr w0, [x29, #228] 0.00 : ffff800010126c80: cmp w0, #0x3 0.00 : ffff800010126c84: b.eq ffff8000101269a4 // b.none : load_balance(): : if (sd->balance_interval < sd->max_interval) 0.00 : ffff800010126c88: ldr x0, [x29, #128] 0.00 : ffff800010126c8c: ldr x1, [x0, #32] 0.00 : ffff800010126c90: ldr w0, [x0, #72] 0.00 : ffff800010126c94: cmp x1, w0, uxtw 0.00 : ffff800010126c98: b.ls ffff800010126d2c // b.plast : sd->balance_interval *= 2; 0.00 : ffff800010126c9c: lsl w0, w0, #1 0.00 : ffff800010126ca0: ldr x1, [x29, #128] 0.00 : ffff800010126ca4: b ffff80001012682c : raw_spin_unlock_irqrestore(&busiest->lock, 0.00 : ffff800010126ca8: mov x1, x0 0.00 : ffff800010126cac: mov x0, x19 0.00 : ffff800010126cb0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : env.flags |= LBF_ALL_PINNED; 0.00 : ffff800010126cb4: ldr w0, [x29, #208] : goto out_one_pinned; 0.00 : ffff800010126cb8: ldp x19, x20, [x29, #16] : env.flags |= LBF_ALL_PINNED; 0.00 : ffff800010126cbc: orr w0, w0, #0x1 : goto out_one_pinned; 0.00 : ffff800010126cc0: ldp x22, x23, [x29, #40] : env.flags |= LBF_ALL_PINNED; 0.00 : ffff800010126cc4: str w0, [x29, #208] : goto out_one_pinned; 0.00 : ffff800010126cc8: ldp x24, x25, [x29, #56] 0.00 : ffff800010126ccc: ldp x27, x28, [x29, #80] 0.00 : ffff800010126cd0: b ffff800010126908 : int *group_imbalance = &sd_parent->groups->sgc->imbalance; 0.00 : ffff800010126cd4: ldr x0, [x1, #16] 12.40 : ffff800010126cd8: ldr x0, [x0, #16] : if (*group_imbalance) 20.76 : ffff800010126cdc: ldr w1, [x0, #40] 0.00 : ffff800010126ce0: cbz w1, ffff800010126900 : *group_imbalance = 0; 0.00 : ffff800010126ce4: str wzr, [x0, #40] 0.00 : ffff800010126ce8: b ffff800010126900 : sched_asym_prefer(): 0.00 : ffff800010126cec: ldp w21, w0, [x29, #160] 0.00 : ffff800010126cf0: bl ffff800010123c30 0.00 : ffff800010126cf4: mov w20, w0 0.00 : ffff800010126cf8: mov w0, w21 0.00 : ffff800010126cfc: bl ffff800010123c30 : asym_active_balance(): : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 0.00 : ffff800010126d00: cmp w20, w0 0.00 : ffff800010126d04: b.gt ffff8000101269a4 : voluntary_active_balance(): : if ((env->idle != CPU_NOT_IDLE) && 0.00 : ffff800010126d08: ldr w0, [x29, #188] 0.00 : ffff800010126d0c: cmp w0, #0x1 0.00 : ffff800010126d10: b.ne ffff800010126c0c // b.any 0.00 : ffff800010126d14: b ffff800010126c7c 1.21 : ffff800010126d18: ldp x19, x20, [x29, #16] 0.00 : ffff800010126d1c: ldp x22, x23, [x29, #40] 1.72 : ffff800010126d20: ldp x24, x25, [x29, #56] 0.00 : ffff800010126d24: ldp x27, x28, [x29, #80] 0.00 : ffff800010126d28: b ffff8000101268f0 0.00 : ffff800010126d2c: ldp x19, x20, [x29, #16] : load_balance(): : sd->nr_balance_failed = sd->cache_nice_tries+1; 0.00 : ffff800010126d30: mov w26, #0x0 // #0 0.00 : ffff800010126d34: ldp x22, x23, [x29, #40] 0.00 : ffff800010126d38: ldp x24, x25, [x29, #56] 0.00 : ffff800010126d3c: ldp x27, x28, [x29, #80] 0.00 : ffff800010126d40: b ffff800010126840 0.00 : ffff800010126d44: ldp x19, x20, [x29, #16] 0.00 : ffff800010126d48: ldp x22, x23, [x29, #40] 0.00 : ffff800010126d4c: ldp x24, x25, [x29, #56] 0.00 : ffff800010126d50: ldp x27, x28, [x29, #80] 0.00 : ffff800010126d54: b ffff800010126900 0.00 : ffff800010126d58: stp x19, x20, [x29, #16] 0.00 : ffff800010126d5c: stp x22, x23, [x29, #40] 0.00 : ffff800010126d60: stp x24, x25, [x29, #56] 0.00 : ffff800010126d64: stp x27, x28, [x29, #80] : } 0.00 : ffff800010126d68: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (54 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101b4820 : : irq_work_needs_cpu(): : : bool irq_work_needs_cpu(void) : { : struct llist_head *raised, *lazy; : : raised = this_cpu_ptr(&raised_list); 1.90 : ffff8000101b4820: adrp x0, ffff800011526000 0.00 : ffff8000101b4824: add x0, x0, #0x78 : lazy = this_cpu_ptr(&lazy_list); 0.00 : ffff8000101b4828: add x1, x0, #0x8 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 1.84 : ffff8000101b482c: mrs x2, tpidr_el1 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 7.56 : ffff8000101b4830: ldr x3, [x0, x2] : irq_work_needs_cpu(): 0.00 : ffff8000101b4834: mov x0, x1 : : if (llist_empty(raised) || arch_irq_work_has_interrupt()) 0.00 : ffff8000101b4838: cbz x3, ffff8000101b4848 : arch_irq_work_has_interrupt(): : : #include : : static inline bool arch_irq_work_has_interrupt(void) : { : return !!__smp_cross_call; 0.00 : ffff8000101b483c: adrp x1, ffff800011af2000 : irq_work_needs_cpu(): 0.00 : ffff8000101b4840: ldr x1, [x1, #2248] 0.00 : ffff8000101b4844: cbz x1, ffff8000101b4854 : __read_once_size(): 79.15 : ffff8000101b4848: ldr x1, [x0, x2] : irq_work_needs_cpu(): : if (llist_empty(lazy)) : return false; 0.00 : ffff8000101b484c: mov w0, #0x0 // #0 : if (llist_empty(lazy)) 0.00 : ffff8000101b4850: cbz x1, ffff8000101b488c : : /* All work should have been flushed before going offline */ : WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); 0.00 : ffff8000101b4854: adrp x0, ffff80001151d000 0.00 : ffff8000101b4858: add x0, x0, #0x18 0.00 : ffff8000101b485c: ldr w2, [x0, x2] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000101b4860: adrp x3, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101b4864: add x3, x3, #0x160 : irq_work_needs_cpu(): : : return true; 0.00 : ffff8000101b4868: mov w0, #0x1 // #1 : test_bit(): 0.00 : ffff8000101b486c: add w1, w2, #0x3f 0.00 : ffff8000101b4870: cmp w2, #0x0 0.00 : ffff8000101b4874: csel w1, w1, w2, lt // lt = tstop 0.00 : ffff8000101b4878: asr w1, w1, #6 0.00 : ffff8000101b487c: sxtw x1, w1 0.00 : ffff8000101b4880: ldr x1, [x3, x1, lsl #3] 0.00 : ffff8000101b4884: lsr x2, x1, x2 : irq_work_needs_cpu(): : WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); 0.00 : ffff8000101b4888: tbz w2, #0, ffff8000101b4890 : } 9.55 : ffff8000101b488c: ret : WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); 0.00 : ffff8000101b4890: brk #0x800 : } 0.00 : ffff8000101b4894: ret Percent | Source code & Disassembly of vmlinux for cycles (51 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011b050 : : calc_global_load_tick(): : */ : void calc_global_load_tick(struct rq *this_rq) : { : long delta; : : if (time_before(jiffies, this_rq->calc_load_update)) 2.08 : ffff80001011b050: adrp x1, ffff800011907000 0.00 : ffff80001011b054: ldr x2, [x1, #2432] 97.92 : ffff80001011b058: ldr x1, [x0, #2856] 0.00 : ffff80001011b05c: sub x2, x2, x1 0.00 : ffff80001011b060: tbnz x2, #63, ffff80001011b084 : calc_load_fold_active(): : nr_active = this_rq->nr_running - adjust; 0.00 : ffff80001011b064: ldr w3, [x0, #4] : nr_active += (long)this_rq->nr_uninterruptible; 0.00 : ffff80001011b068: ldr x4, [x0, #2344] : if (nr_active != this_rq->calc_load_active) { 0.00 : ffff80001011b06c: ldr x2, [x0, #2864] : nr_active += (long)this_rq->nr_uninterruptible; 0.00 : ffff80001011b070: add x3, x3, x4 : if (nr_active != this_rq->calc_load_active) { 0.00 : ffff80001011b074: cmp x3, x2 0.00 : ffff80001011b078: b.ne ffff80001011b088 // b.any : calc_global_load_tick(): : : delta = calc_load_fold_active(this_rq, 0); : if (delta) : atomic_long_add(delta, &calc_load_tasks); : : this_rq->calc_load_update += LOAD_FREQ; 0.00 : ffff80001011b07c: add x1, x1, #0x4e3 0.00 : ffff80001011b080: str x1, [x0, #2856] : } 0.00 : ffff80001011b084: ret : calc_load_fold_active(): : delta = nr_active - this_rq->calc_load_active; 0.00 : ffff80001011b088: sub x2, x3, x2 : this_rq->calc_load_active = nr_active; 0.00 : ffff80001011b08c: str x3, [x0, #2864] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011b090: b ffff80001011b0b4 0.00 : ffff80001011b094: b ffff80001011b0b4 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff80001011b098: adrp x3, ffff800011afb000 0.00 : ffff80001011b09c: add x3, x3, #0xf48 0.00 : ffff80001011b0a0: add x4, x3, #0x38 0.00 : ffff80001011b0a4: stadd x2, [x4] : calc_global_load_tick(): : this_rq->calc_load_update += LOAD_FREQ; 0.00 : ffff80001011b0a8: add x1, x1, #0x4e3 0.00 : ffff80001011b0ac: str x1, [x0, #2856] 0.00 : ffff80001011b0b0: b ffff80001011b084 : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff80001011b0b4: adrp x3, ffff800011afb000 0.00 : ffff80001011b0b8: add x3, x3, #0xf48 0.00 : ffff80001011b0bc: add x3, x3, #0x38 0.00 : ffff80001011b0c0: b ffff80001011b100 : calc_global_load_tick(): 0.00 : ffff80001011b0c4: add x1, x1, #0x4e3 0.00 : ffff80001011b0c8: str x1, [x0, #2856] 0.00 : ffff80001011b0cc: b ffff80001011b084 Percent | Source code & Disassembly of vmlinux for cycles (50 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016de90 : : timekeeping_advance(): : /* : * timekeeping_advance - Updates the timekeeper to the current time and : * current NTP tick length : */ : static void timekeeping_advance(enum timekeeping_adv_mode mode) : { 0.00 : ffff80001016de90: stp x29, x30, [sp, #-144]! 0.00 : ffff80001016de94: mov x29, sp 0.00 : ffff80001016de98: stp x20, x21, [sp, #24] 0.00 : ffff80001016de9c: adrp x21, ffff800011909000 0.00 : ffff80001016dea0: stp x22, x23, [sp, #40] 0.00 : ffff80001016dea4: add x1, x21, #0x908 0.00 : ffff80001016dea8: stp x24, x25, [sp, #56] : u64 offset; : int shift = 0, maxshift; : unsigned int clock_set = 0; : unsigned long flags; : : raw_spin_lock_irqsave(&timekeeper_lock, flags); 0.00 : ffff80001016deac: adrp x20, ffff800011b20000 <__log_buf+0x1fc10> 0.00 : ffff80001016deb0: add x22, x20, #0xec0 : { 0.00 : ffff80001016deb4: mov w23, w0 : raw_spin_lock_irqsave(&timekeeper_lock, flags); 0.00 : ffff80001016deb8: add x24, x22, #0x160 : { 0.00 : ffff80001016debc: ldr x0, [x1] 0.00 : ffff80001016dec0: str x0, [x29, #136] 0.00 : ffff80001016dec4: mov x0, #0x0 // #0 : raw_spin_lock_irqsave(&timekeeper_lock, flags); 0.00 : ffff80001016dec8: mov x0, x24 0.00 : ffff80001016decc: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff80001016ded0: mov x25, x0 : : /* Make sure we're fully resumed: */ : if (unlikely(timekeeping_suspended)) 0.00 : ffff80001016ded4: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001016ded8: ldr w0, [x0, #708] 0.00 : ffff80001016dedc: cbnz w0, ffff80001016e38c 0.00 : ffff80001016dee0: str x19, [x29, #16] : #else : offset = clocksource_delta(tk_clock_read(&tk->tkr_mono), : tk->tkr_mono.cycle_last, tk->tkr_mono.mask); : : /* Check if there's really nothing to do */ : if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK) 0.00 : ffff80001016dee4: eor w23, w23, #0x1 0.00 : ffff80001016dee8: str x26, [x29, #72] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001016deec: add x26, x22, #0x170 0.00 : ffff80001016def0: ldr x2, [x22, #368] : tk_clock_read(): : return clock->read(clock); 0.00 : ffff80001016def4: mov x0, x2 2.08 : ffff80001016def8: ldr x2, [x2] 0.00 : ffff80001016defc: blr x2 : clocksource_delta(): : return ret & ~(mask >> 1) ? 0 : ret; : } : #else : static inline u64 clocksource_delta(u64 now, u64 last, u64 mask) : { : return (now - last) & mask; 0.00 : ffff80001016df00: ldp x19, x2, [x26, #8] : timekeeping_advance(): : if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK) 2.14 : ffff80001016df04: ldr x3, [x22, #288] : clocksource_delta(): 0.00 : ffff80001016df08: sub x0, x0, x2 0.00 : ffff80001016df0c: and x19, x0, x19 : timekeeping_advance(): 0.00 : ffff80001016df10: cmp x3, x19 0.00 : ffff80001016df14: cset w0, hi // hi = pmore 0.00 : ffff80001016df18: tst w23, w0 0.00 : ffff80001016df1c: b.ne ffff80001016e340 // b.any 0.00 : ffff80001016df20: stp x27, x28, [x29, #80] 0.00 : ffff80001016df24: clz x24, x19 0.00 : ffff80001016df28: mov x27, #0x3f // #63 0.00 : ffff80001016df2c: cmp x19, #0x0 : * we calculate the largest doubling multiple of cycle_intervals : * that is smaller than the offset. We then accumulate that : * chunk in one go, and then try to consume the next smaller : * doubled multiple. : */ : shift = ilog2(offset) - ilog2(tk->cycle_interval); 0.00 : ffff80001016df30: ldr x2, [x26, #216] 0.00 : ffff80001016df34: sub x24, x27, x24 0.00 : ffff80001016df38: mov w28, #0xffffffff // #-1 0.00 : ffff80001016df3c: csel w24, w28, w24, eq // eq = none 0.00 : ffff80001016df40: clz x0, x2 0.00 : ffff80001016df44: cmp x2, #0x0 0.00 : ffff80001016df48: sub x0, x27, x0 : accumulate_nsecs_to_secs(): : u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 0.00 : ffff80001016df4c: mov x23, #0xca00 // #51712 0.00 : ffff80001016df50: csel w28, w28, w0, eq // eq = none 0.00 : ffff80001016df54: movk x23, #0x3b9a, lsl #16 : timekeeping_advance(): : shift = max(0, shift); 0.00 : ffff80001016df58: subs w24, w24, w28 : unsigned int clock_set = 0; 0.00 : ffff80001016df5c: mov w22, #0x0 // #0 : shift = max(0, shift); 0.00 : ffff80001016df60: csel w24, w24, wzr, pl // pl = nfrst : /* Bound shift to one less than what overflows tick_length */ : maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; 0.00 : ffff80001016df64: bl ffff80001016f138 0.00 : ffff80001016df68: clz x28, x0 0.00 : ffff80001016df6c: sub w28, w28, w27 0.00 : ffff80001016df70: cmp x0, #0x0 0.00 : ffff80001016df74: add w28, w28, #0x3e 0.00 : ffff80001016df78: ldr x1, [x26, #216] 0.00 : ffff80001016df7c: csel w28, w27, w28, eq // eq = none : shift = min(shift, maxshift); 0.00 : ffff80001016df80: cmp w28, w24 0.00 : ffff80001016df84: csel w28, w28, w24, le : while (offset >= tk->cycle_interval) { 0.00 : ffff80001016df88: b ffff80001016df98 : offset = logarithmic_accumulation(tk, offset, shift, : &clock_set); : if (offset < tk->cycle_interval<= tk->cycle_interval) { 0.00 : ffff80001016df98: cmp x1, x19 0.00 : ffff80001016df9c: b.hi ffff80001016e130 // b.pmore : logarithmic_accumulation(): : u64 interval = tk->cycle_interval << shift; 0.00 : ffff80001016dfa0: lsl x2, x1, x28 : if (offset < interval) 0.00 : ffff80001016dfa4: cmp x19, x2 0.00 : ffff80001016dfa8: b.cc ffff80001016df8c // b.lo, b.ul, b.last : tk->tkr_mono.cycle_last += interval; 0.00 : ffff80001016dfac: add x0, x20, #0xec0 : offset -= interval; 0.00 : ffff80001016dfb0: sub x19, x19, x2 : tk->tkr_mono.cycle_last += interval; 0.00 : ffff80001016dfb4: add x0, x0, #0x170 : accumulate_nsecs_to_secs(): : unsigned int clock_set = 0; 0.00 : ffff80001016dfb8: mov w24, #0x0 // #0 : tk->tkr_mono.xtime_nsec -= nsecps; 0.00 : ffff80001016dfbc: mov x27, x0 : u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 0.00 : ffff80001016dfc0: ldr w3, [x0, #28] : logarithmic_accumulation(): : tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; 0.00 : ffff80001016dfc4: ldr x1, [x0, #224] : tk->tkr_mono.cycle_last += interval; 0.00 : ffff80001016dfc8: ldr x5, [x0, #16] : tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; 0.00 : ffff80001016dfcc: ldr x6, [x0, #32] : accumulate_nsecs_to_secs(): : u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 0.00 : ffff80001016dfd0: lsl x26, x23, x3 : logarithmic_accumulation(): : tk->tkr_raw.cycle_last += interval; 0.00 : ffff80001016dfd4: ldr x4, [x0, #72] : tk->tkr_mono.cycle_last += interval; 0.00 : ffff80001016dfd8: add x5, x5, x2 : tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; 0.00 : ffff80001016dfdc: lsl x1, x1, x28 : tk->tkr_mono.cycle_last += interval; 0.00 : ffff80001016dfe0: str x5, [x0, #16] : tk->tkr_raw.cycle_last += interval; 0.00 : ffff80001016dfe4: add x2, x4, x2 : tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; 0.00 : ffff80001016dfe8: add x1, x1, x6 0.00 : ffff80001016dfec: str x1, [x0, #32] : tk->tkr_raw.cycle_last += interval; 0.00 : ffff80001016dff0: str x2, [x0, #72] : accumulate_nsecs_to_secs(): : while (tk->tkr_mono.xtime_nsec >= nsecps) { 0.00 : ffff80001016dff4: cmp x26, x1 0.00 : ffff80001016dff8: b.hi ffff80001016e030 // b.pmore : tk->xtime_sec++; 0.00 : ffff80001016dffc: ldr x0, [x27, #112] : tk->tkr_mono.xtime_nsec -= nsecps; 0.00 : ffff80001016e000: sub x1, x1, x26 : if (unlikely(tk->skip_second_overflow)) { 0.00 : ffff80001016e004: ldr w2, [x27, #272] : tk->xtime_sec++; 0.00 : ffff80001016e008: add x0, x0, #0x1 : tk->tkr_mono.xtime_nsec -= nsecps; 0.00 : ffff80001016e00c: str x1, [x27, #32] : tk->xtime_sec++; 0.00 : ffff80001016e010: str x0, [x27, #112] : if (unlikely(tk->skip_second_overflow)) { 0.00 : ffff80001016e014: cbnz w2, ffff80001016e0c8 : leap = second_overflow(tk->xtime_sec); 0.00 : ffff80001016e018: bl ffff80001016f198 0.00 : ffff80001016e01c: mov w4, w0 : if (unlikely(leap)) { 0.00 : ffff80001016e020: cbnz w0, ffff80001016e0d0 0.00 : ffff80001016e024: ldr x1, [x27, #32] : while (tk->tkr_mono.xtime_nsec >= nsecps) { 0.00 : ffff80001016e028: cmp x26, x1 0.00 : ffff80001016e02c: b.ls ffff80001016dffc // b.plast : logarithmic_accumulation(): : tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; 0.00 : ffff80001016e030: add x2, x20, #0xec0 : *clock_set |= accumulate_nsecs_to_secs(tk); 0.00 : ffff80001016e034: orr w22, w22, w24 : tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; 0.00 : ffff80001016e038: add x2, x2, #0x170 0.00 : ffff80001016e03c: ldr x0, [x2, #240] 0.00 : ffff80001016e040: ldr x3, [x2, #88] : snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; 0.00 : ffff80001016e044: ldr w1, [x2, #84] : tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; 0.00 : ffff80001016e048: lsl x0, x0, x28 0.00 : ffff80001016e04c: add x0, x0, x3 0.00 : ffff80001016e050: str x0, [x2, #88] : snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; 0.00 : ffff80001016e054: lsl x1, x23, x1 : while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { 0.00 : ffff80001016e058: cmp x0, x1 0.00 : ffff80001016e05c: b.cc ffff80001016e088 // b.lo, b.ul, b.last 0.00 : ffff80001016e060: ldr x2, [x2, #192] 0.00 : ffff80001016e064: nop : tk->tkr_raw.xtime_nsec -= snsec_per_sec; 0.00 : ffff80001016e068: sub x0, x0, x1 : tk->raw_sec++; 0.00 : ffff80001016e06c: add x2, x2, #0x1 : while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { 0.00 : ffff80001016e070: cmp x1, x0 0.00 : ffff80001016e074: b.ls ffff80001016e068 // b.plast 0.00 : ffff80001016e078: add x1, x20, #0xec0 0.00 : ffff80001016e07c: str x0, [x1, #456] 0.00 : ffff80001016e080: str x2, [x1, #560] 0.00 : ffff80001016e084: nop : tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << 0.00 : ffff80001016e088: add x0, x20, #0xec0 : (tk->ntp_error_shift + shift); 0.00 : ffff80001016e08c: ldr w2, [x0, #632] : tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << 0.00 : ffff80001016e090: ldr x4, [x0, #592] 0.00 : ffff80001016e094: ldr x6, [x0, #600] : (tk->ntp_error_shift + shift); 0.00 : ffff80001016e098: add w2, w28, w2 : tk->ntp_error += tk->ntp_tick << shift; 0.00 : ffff80001016e09c: ldr x3, [x0, #616] 0.00 : ffff80001016e0a0: ldr x1, [x0, #584] : tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << 0.00 : ffff80001016e0a4: add x4, x4, x6 : tk->ntp_error += tk->ntp_tick << shift; 0.00 : ffff80001016e0a8: ldr x5, [x0, #624] 0.00 : ffff80001016e0ac: lsl x3, x3, x28 0.00 : ffff80001016e0b0: add x3, x3, x5 : tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << 0.00 : ffff80001016e0b4: lsl x4, x4, x2 0.00 : ffff80001016e0b8: lsl x2, x1, x28 0.00 : ffff80001016e0bc: sub x3, x3, x4 0.00 : ffff80001016e0c0: str x3, [x0, #624] 0.00 : ffff80001016e0c4: b ffff80001016df8c : accumulate_nsecs_to_secs(): : tk->skip_second_overflow = 0; 0.00 : ffff80001016e0c8: str wzr, [x27, #272] 0.00 : ffff80001016e0cc: b ffff80001016dff4 : tk->xtime_sec += leap; 0.00 : ffff80001016e0d0: ldr x5, [x27, #112] 0.00 : ffff80001016e0d4: sxtw x1, w0 : timespec64_sub(): : */ : static inline struct timespec64 timespec64_sub(struct timespec64 lhs, : struct timespec64 rhs) : { : struct timespec64 ts_delta; : set_normalized_timespec64(&ts_delta, lhs.tv_sec - rhs.tv_sec, 0.00 : ffff80001016e0d8: ldp x6, x2, [x27, #128] : accumulate_nsecs_to_secs(): 0.00 : ffff80001016e0dc: add x5, x5, x1 0.00 : ffff80001016e0e0: str w4, [x29, #108] : timespec64_sub(): 0.00 : ffff80001016e0e4: add x0, x29, #0x78 0.00 : ffff80001016e0e8: sub x1, x6, x1 : accumulate_nsecs_to_secs(): 0.00 : ffff80001016e0ec: str x5, [x27, #112] : timespec64_sub(): 0.00 : ffff80001016e0f0: bl ffff800010167138 : accumulate_nsecs_to_secs(): : clock_set = TK_CLOCK_WAS_SET; 0.00 : ffff80001016e0f4: mov w24, #0x4 // #4 : tk_set_wall_to_mono(tk, 0.00 : ffff80001016e0f8: ldp x1, x2, [x29, #120] 0.00 : ffff80001016e0fc: mov x0, x27 0.00 : ffff80001016e100: bl ffff80001016ce88 : __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); 0.00 : ffff80001016e104: ldr w0, [x27, #168] 0.00 : ffff80001016e108: ldr w4, [x29, #108] : __timekeeping_set_tai_offset(): : tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 0.00 : ffff80001016e10c: ldr x2, [x27, #144] : accumulate_nsecs_to_secs(): : __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); 0.00 : ffff80001016e110: sub w4, w0, w4 : __timekeeping_set_tai_offset(): : tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 0.00 : ffff80001016e114: mov w0, #0xca00 // #51712 0.00 : ffff80001016e118: movk w0, #0x3b9a, lsl #16 : tk->tai_offset = tai_offset; 0.00 : ffff80001016e11c: str w4, [x27, #168] 0.00 : ffff80001016e120: ldr x1, [x27, #32] : tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 0.00 : ffff80001016e124: smaddl x4, w4, w0, x2 0.00 : ffff80001016e128: str x4, [x27, #160] 0.00 : ffff80001016e12c: b ffff80001016dff4 : timekeeping_adjust(): : if (likely(tk->ntp_tick == ntp_tick_length())) { 0.00 : ffff80001016e130: add x23, x20, #0xec0 0.00 : ffff80001016e134: add x23, x23, #0x170 0.00 : ffff80001016e138: ldr x24, [x23, #248] 0.00 : ffff80001016e13c: bl ffff80001016f138 0.00 : ffff80001016e140: cmp x24, x0 0.00 : ffff80001016e144: b.ne ffff80001016e39c // b.any : mult = tk->tkr_mono.mult - tk->ntp_err_mult; 0.00 : ffff80001016e148: ldr w1, [x23, #24] 0.00 : ffff80001016e14c: ldr w0, [x23, #268] 0.00 : ffff80001016e150: ldr x4, [x23, #216] 0.00 : ffff80001016e154: sub w3, w1, w0 : tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0; 0.00 : ffff80001016e158: add x2, x20, #0xec0 0.00 : ffff80001016e15c: ldr x0, [x2, #624] 0.00 : ffff80001016e160: cmp x0, #0x0 0.00 : ffff80001016e164: cset w0, gt 0.00 : ffff80001016e168: str w0, [x2, #636] : mult += tk->ntp_err_mult; 0.00 : ffff80001016e16c: add w0, w0, w3 : timekeeping_apply_adjustment(): : if (mult_adj == 0) { 0.00 : ffff80001016e170: subs w1, w0, w1 0.00 : ffff80001016e174: b.ne ffff80001016e28c // b.any : timekeeping_adjust(): : if (unlikely(tk->tkr_mono.clock->maxadj && 0.00 : ffff80001016e178: add x0, x20, #0xec0 0.00 : ffff80001016e17c: add x2, x0, #0x170 0.00 : ffff80001016e180: ldr x1, [x0, #368] 0.00 : ffff80001016e184: ldr w0, [x1, #32] 0.00 : ffff80001016e188: cbnz w0, ffff80001016e3c4 : if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { 0.00 : ffff80001016e18c: add x0, x20, #0xec0 0.00 : ffff80001016e190: mov x26, #0xca00 // #51712 0.00 : ffff80001016e194: movk x26, #0x3b9a, lsl #16 0.00 : ffff80001016e198: add x0, x0, #0x170 0.00 : ffff80001016e19c: ldr x1, [x0, #32] 0.00 : ffff80001016e1a0: ldr w2, [x0, #28] 0.00 : ffff80001016e1a4: lsl x26, x26, x2 0.00 : ffff80001016e1a8: tbnz x1, #63, ffff80001016e410 : accumulate_nsecs_to_secs(): : tk->tkr_mono.xtime_nsec -= nsecps; 0.00 : ffff80001016e1ac: add x19, x20, #0xec0 : __timekeeping_set_tai_offset(): : tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 0.00 : ffff80001016e1b0: mov w24, #0xca00 // #51712 : accumulate_nsecs_to_secs(): : tk->tkr_mono.xtime_nsec -= nsecps; 0.00 : ffff80001016e1b4: add x19, x19, #0x170 : unsigned int clock_set = 0; 0.00 : ffff80001016e1b8: mov w23, #0x0 // #0 : __timekeeping_set_tai_offset(): : tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 0.00 : ffff80001016e1bc: movk w24, #0x3b9a, lsl #16 : accumulate_nsecs_to_secs(): : while (tk->tkr_mono.xtime_nsec >= nsecps) { 0.00 : ffff80001016e1c0: cmp x26, x1 0.00 : ffff80001016e1c4: b.hi ffff80001016e1fc // b.pmore : tk->xtime_sec++; 0.00 : ffff80001016e1c8: ldr x0, [x19, #112] : tk->tkr_mono.xtime_nsec -= nsecps; 0.00 : ffff80001016e1cc: sub x1, x1, x26 : if (unlikely(tk->skip_second_overflow)) { 0.00 : ffff80001016e1d0: ldr w2, [x19, #272] : tk->xtime_sec++; 0.00 : ffff80001016e1d4: add x0, x0, #0x1 : tk->tkr_mono.xtime_nsec -= nsecps; 0.00 : ffff80001016e1d8: str x1, [x19, #32] : tk->xtime_sec++; 0.00 : ffff80001016e1dc: str x0, [x19, #112] : if (unlikely(tk->skip_second_overflow)) { 0.00 : ffff80001016e1e0: cbnz w2, ffff80001016e2e8 : leap = second_overflow(tk->xtime_sec); 0.00 : ffff80001016e1e4: bl ffff80001016f198 0.00 : ffff80001016e1e8: mov w27, w0 : if (unlikely(leap)) { 0.00 : ffff80001016e1ec: cbnz w0, ffff80001016e2f0 0.00 : ffff80001016e1f0: ldr x1, [x19, #32] : while (tk->tkr_mono.xtime_nsec >= nsecps) { 0.00 : ffff80001016e1f4: cmp x26, x1 0.00 : ffff80001016e1f8: b.ls ffff80001016e1c8 // b.plast : raw_write_seqcount_begin(): : : : : static inline void raw_write_seqcount_begin(seqcount_t *s) : { : s->sequence++; 0.00 : ffff80001016e1fc: add x20, x20, #0xec0 : timekeeping_advance(): : : /* : * Finally, make sure that after the rounding : * xtime_nsec isn't larger than NSEC_PER_SEC : */ : clock_set |= accumulate_nsecs_to_secs(tk); 0.00 : ffff80001016e200: orr w22, w23, w22 : raw_write_seqcount_begin(): 0.00 : ffff80001016e204: ldr w0, [x20, #64] 0.00 : ffff80001016e208: add w0, w0, #0x1 0.00 : ffff80001016e20c: str w0, [x20, #64] : smp_wmb(); 0.00 : ffff80001016e210: dmb ishst : timekeeping_advance(): : * well, i.e. move the timekeeper pointer getter into the : * spinlocked/seqcount protected sections. And we trade this : * memcpy under the tk_core.seq against one before we start : * updating. : */ : timekeeping_update(tk, clock_set); 52.91 : ffff80001016e214: add x19, x20, #0x170 0.00 : ffff80001016e218: mov w1, w22 0.00 : ffff80001016e21c: mov x0, x19 0.00 : ffff80001016e220: bl ffff80001016d070 : memcpy(real_tk, tk, sizeof(*tk)); 0.00 : ffff80001016e224: mov x2, #0x118 // #280 0.00 : ffff80001016e228: mov x1, x19 0.00 : ffff80001016e22c: add x0, x20, #0x48 0.00 : ffff80001016e230: bl ffff800010cb7dc0 <__memcpy> : raw_write_seqcount_end(): : } : : static inline void raw_write_seqcount_end(seqcount_t *s) : { : smp_wmb(); 0.00 : ffff80001016e234: dmb ishst : s->sequence++; 38.78 : ffff80001016e238: ldr w2, [x20, #64] : timekeeping_advance(): : /* The memcpy must come last. Do not put anything here! */ : write_seqcount_end(&tk_core.seq); : out: : raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 0.00 : ffff80001016e23c: mov x1, x25 0.00 : ffff80001016e240: add x0, x20, #0x160 : raw_write_seqcount_end(): 0.00 : ffff80001016e244: add w2, w2, #0x1 0.00 : ffff80001016e248: str w2, [x20, #64] : timekeeping_advance(): 0.00 : ffff80001016e24c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : if (clock_set) 0.00 : ffff80001016e250: cbz w22, ffff80001016e2d8 : /* Have to call _delayed version, since in irq context*/ : clock_was_set_delayed(); 0.00 : ffff80001016e254: bl ffff80001016bb00 0.00 : ffff80001016e258: ldr x19, [x29, #16] 0.00 : ffff80001016e25c: ldp x26, x27, [x29, #72] 0.00 : ffff80001016e260: ldr x28, [x29, #88] : } 0.00 : ffff80001016e264: add x21, x21, #0x908 0.00 : ffff80001016e268: ldr x1, [x29, #136] 0.00 : ffff80001016e26c: ldr x0, [x21] 0.00 : ffff80001016e270: eor x0, x1, x0 0.00 : ffff80001016e274: cbnz x0, ffff80001016e430 0.00 : ffff80001016e278: ldp x20, x21, [sp, #24] 0.00 : ffff80001016e27c: ldp x22, x23, [sp, #40] 0.00 : ffff80001016e280: ldp x24, x25, [sp, #56] 0.00 : ffff80001016e284: ldp x29, x30, [sp], #144 0.00 : ffff80001016e288: ret : timekeeping_apply_adjustment(): : } else if (mult_adj == -1) { 0.00 : ffff80001016e28c: cmn w1, #0x1 0.00 : ffff80001016e290: b.eq ffff80001016e360 // b.none : } else if (mult_adj != 1) { 0.00 : ffff80001016e294: cmp w1, #0x1 0.00 : ffff80001016e298: b.eq ffff80001016e2a8 // b.none : interval *= mult_adj; 0.00 : ffff80001016e29c: sxtw x2, w1 0.00 : ffff80001016e2a0: mul x4, x4, x2 : offset *= mult_adj; 0.00 : ffff80001016e2a4: mul x19, x19, x2 : if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { 0.00 : ffff80001016e2a8: cmp w1, #0x0 0.00 : ffff80001016e2ac: ccmp w0, w1, #0x2, gt 0.00 : ffff80001016e2b0: b.cc ffff80001016e358 // b.lo, b.ul, b.last : tk->tkr_mono.mult += mult_adj; 0.00 : ffff80001016e2b4: add x1, x20, #0xec0 : tk->tkr_mono.xtime_nsec -= offset; 0.00 : ffff80001016e2b8: ldr x3, [x1, #400] : tk->xtime_interval += interval; 0.00 : ffff80001016e2bc: ldr x2, [x1, #592] : tk->tkr_mono.xtime_nsec -= offset; 0.00 : ffff80001016e2c0: sub x19, x3, x19 : tk->tkr_mono.mult += mult_adj; 0.00 : ffff80001016e2c4: str w0, [x1, #392] : tk->xtime_interval += interval; 0.00 : ffff80001016e2c8: add x4, x2, x4 : tk->tkr_mono.xtime_nsec -= offset; 0.00 : ffff80001016e2cc: str x19, [x1, #400] : tk->xtime_interval += interval; 0.00 : ffff80001016e2d0: str x4, [x1, #592] 0.00 : ffff80001016e2d4: b ffff80001016e178 0.00 : ffff80001016e2d8: ldr x19, [x29, #16] 0.00 : ffff80001016e2dc: ldp x26, x27, [x29, #72] 0.00 : ffff80001016e2e0: ldr x28, [x29, #88] 0.00 : ffff80001016e2e4: b ffff80001016e264 : accumulate_nsecs_to_secs(): : tk->skip_second_overflow = 0; 0.00 : ffff80001016e2e8: str wzr, [x19, #272] 0.00 : ffff80001016e2ec: b ffff80001016e1c0 : tk->xtime_sec += leap; 0.00 : ffff80001016e2f0: ldr x3, [x19, #112] 0.00 : ffff80001016e2f4: sxtw x1, w0 : timespec64_sub(): 0.00 : ffff80001016e2f8: ldp x4, x2, [x19, #128] : accumulate_nsecs_to_secs(): 0.00 : ffff80001016e2fc: add x3, x3, x1 : timespec64_sub(): 0.00 : ffff80001016e300: add x0, x29, #0x78 : accumulate_nsecs_to_secs(): 0.00 : ffff80001016e304: str x3, [x19, #112] : timespec64_sub(): 0.00 : ffff80001016e308: sub x1, x4, x1 : accumulate_nsecs_to_secs(): : clock_set = TK_CLOCK_WAS_SET; 0.00 : ffff80001016e30c: mov w23, #0x4 // #4 : timespec64_sub(): 0.00 : ffff80001016e310: bl ffff800010167138 : accumulate_nsecs_to_secs(): : tk_set_wall_to_mono(tk, 0.00 : ffff80001016e314: ldp x1, x2, [x29, #120] 0.00 : ffff80001016e318: mov x0, x19 0.00 : ffff80001016e31c: bl ffff80001016ce88 : __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); 0.00 : ffff80001016e320: ldr w0, [x19, #168] : __timekeeping_set_tai_offset(): : tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 0.00 : ffff80001016e324: ldr x2, [x19, #144] : accumulate_nsecs_to_secs(): : __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); 0.00 : ffff80001016e328: sub w27, w0, w27 : __timekeeping_set_tai_offset(): : tk->tai_offset = tai_offset; 0.00 : ffff80001016e32c: str w27, [x19, #168] 0.00 : ffff80001016e330: ldr x1, [x19, #32] : tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 0.00 : ffff80001016e334: smaddl x27, w27, w24, x2 0.00 : ffff80001016e338: str x27, [x19, #160] 0.00 : ffff80001016e33c: b ffff80001016e1c0 : timekeeping_advance(): : raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 0.00 : ffff80001016e340: mov x1, x25 0.00 : ffff80001016e344: mov x0, x24 0.00 : ffff80001016e348: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff80001016e34c: ldr x19, [x29, #16] 0.00 : ffff80001016e350: ldr x26, [x29, #72] 0.00 : ffff80001016e354: b ffff80001016e264 : timekeeping_apply_adjustment(): : WARN_ON_ONCE(1); 0.00 : ffff80001016e358: brk #0x800 0.00 : ffff80001016e35c: b ffff80001016e178 : tk->tkr_mono.mult += mult_adj; 0.00 : ffff80001016e360: add x1, x20, #0xec0 : interval = -interval; 0.00 : ffff80001016e364: neg x4, x4 : offset = -offset; 0.00 : ffff80001016e368: neg x19, x19 : tk->tkr_mono.xtime_nsec -= offset; 0.00 : ffff80001016e36c: ldr x3, [x1, #400] : tk->xtime_interval += interval; 0.00 : ffff80001016e370: ldr x2, [x1, #592] : tk->tkr_mono.xtime_nsec -= offset; 0.00 : ffff80001016e374: sub x19, x3, x19 : tk->tkr_mono.mult += mult_adj; 0.00 : ffff80001016e378: str w0, [x1, #392] : tk->xtime_interval += interval; 0.00 : ffff80001016e37c: add x4, x2, x4 : tk->tkr_mono.xtime_nsec -= offset; 0.00 : ffff80001016e380: str x19, [x1, #400] : tk->xtime_interval += interval; 0.00 : ffff80001016e384: str x4, [x1, #592] 0.00 : ffff80001016e388: b ffff80001016e178 : timekeeping_advance(): : raw_spin_unlock_irqrestore(&timekeeper_lock, flags); 0.00 : ffff80001016e38c: mov x1, x25 0.00 : ffff80001016e390: mov x0, x24 0.00 : ffff80001016e394: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff80001016e398: b ffff80001016e264 : timekeeping_adjust(): : tk->ntp_tick = ntp_tick_length(); 0.00 : ffff80001016e39c: bl ffff80001016f138 0.00 : ffff80001016e3a0: str x0, [x23, #248] : mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) - 0.00 : ffff80001016e3a4: ldr w2, [x23, #264] 0.00 : ffff80001016e3a8: ldr x3, [x23, #232] 0.00 : ffff80001016e3ac: ldr x4, [x23, #216] 0.00 : ffff80001016e3b0: lsr x0, x0, x2 0.00 : ffff80001016e3b4: sub x0, x0, x3 0.00 : ffff80001016e3b8: ldr w1, [x23, #24] 0.00 : ffff80001016e3bc: udiv x3, x0, x4 0.00 : ffff80001016e3c0: b ffff80001016e158 : if (unlikely(tk->tkr_mono.clock->maxadj && 4.09 : ffff80001016e3c4: ldr w3, [x1, #16] 0.00 : ffff80001016e3c8: ldr w2, [x2, #24] 0.00 : ffff80001016e3cc: subs w4, w2, w3 0.00 : ffff80001016e3d0: cneg w4, w4, mi // mi = first 0.00 : ffff80001016e3d4: cmp w0, w4 0.00 : ffff80001016e3d8: b.cs ffff80001016e18c // b.hs, b.nlast : printk_once(KERN_WARNING 0.00 : ffff80001016e3dc: adrp x4, ffff800011ad8000 0.00 : ffff80001016e3e0: ldrb w5, [x4, #1227] 0.00 : ffff80001016e3e4: cbnz w5, ffff80001016e18c 0.00 : ffff80001016e3e8: ldr x1, [x1, #48] 0.00 : ffff80001016e3ec: mov w6, #0x1 // #1 0.00 : ffff80001016e3f0: adrp x5, ffff8000111c8000 0.00 : ffff80001016e3f4: mov w0, w0 0.00 : ffff80001016e3f8: add x3, x0, w3, uxtw 0.00 : ffff80001016e3fc: strb w6, [x4, #1227] 0.00 : ffff80001016e400: mov w2, w2 0.00 : ffff80001016e404: add x0, x5, #0xb60 0.00 : ffff80001016e408: bl ffff80001014a85c 0.00 : ffff80001016e40c: b ffff80001016e18c : tk->xtime_sec--; 0.00 : ffff80001016e410: ldr x2, [x0, #112] : tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC << 0.00 : ffff80001016e414: add x1, x1, x26 : tk->skip_second_overflow = 1; 0.00 : ffff80001016e418: mov w3, #0x1 // #1 : tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC << 0.00 : ffff80001016e41c: str x1, [x0, #32] : tk->xtime_sec--; 0.00 : ffff80001016e420: sub x2, x2, #0x1 0.00 : ffff80001016e424: str x2, [x0, #112] : tk->skip_second_overflow = 1; 0.00 : ffff80001016e428: str w3, [x0, #272] 0.00 : ffff80001016e42c: b ffff80001016e1ac 0.00 : ffff80001016e430: str x19, [x29, #16] 0.00 : ffff80001016e434: stp x26, x27, [x29, #72] 0.00 : ffff80001016e438: str x28, [x29, #88] : timekeeping_advance(): : } 0.00 : ffff80001016e43c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (48 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016bcb0 : : hrtimer_interrupt(): : /* : * High resolution timer interrupt : * Called with interrupts disabled : */ : void hrtimer_interrupt(struct clock_event_device *dev) : { 8.43 : ffff80001016bcb0: stp x29, x30, [sp, #-96]! 0.00 : ffff80001016bcb4: mov x29, sp 4.33 : ffff80001016bcb8: str x19, [sp, #16] : struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016bcbc: adrp x19, ffff800011525000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 4.05 : ffff80001016bcc0: mrs x1, tpidr_el1 : hrtimer_interrupt(): 0.00 : ffff80001016bcc4: add x19, x19, #0x980 0.00 : ffff80001016bcc8: add x19, x19, x1 : ktime_t expires_next, now, entry_time, delta; : unsigned long flags; : int retries = 0; : : BUG_ON(!cpu_base->hres_active); 0.00 : ffff80001016bccc: ldrb w1, [x19, #16] 50.13 : ffff80001016bcd0: stp x20, x21, [x29, #24] 0.00 : ffff80001016bcd4: stp x22, x23, [x29, #40] 0.00 : ffff80001016bcd8: stp x24, x25, [x29, #56] 0.00 : ffff80001016bcdc: stp x26, x27, [x29, #72] 0.00 : ffff80001016bce0: str x28, [x29, #88] 0.00 : ffff80001016bce4: tbz w1, #0, ffff80001016bedc 0.00 : ffff80001016bce8: mov x1, x0 : cpu_base->nr_events++; 0.00 : ffff80001016bcec: ldr w2, [x19, #20] : dev->next_event = KTIME_MAX; 0.00 : ffff80001016bcf0: mov x22, #0x7fffffffffffffff // #9223372036854775807 : : raw_spin_lock_irqsave(&cpu_base->lock, flags); 0.00 : ffff80001016bcf4: mov x0, x19 : cpu_base->nr_events++; 0.00 : ffff80001016bcf8: add w2, w2, #0x1 4.24 : ffff80001016bcfc: str w2, [x19, #20] : dev->next_event = KTIME_MAX; 0.00 : ffff80001016bd00: str x22, [x1, #24] : hrtimer_update_base(): : ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; 0.00 : ffff80001016bd04: add x25, x19, #0xb8 : ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; 0.00 : ffff80001016bd08: add x26, x19, #0xf8 : ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; 0.00 : ffff80001016bd0c: add x27, x19, #0x138 : hrtimer_interrupt(): : raw_spin_lock_irqsave(&cpu_base->lock, flags); 0.00 : ffff80001016bd10: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : hrtimer_update_base(): : ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 0.00 : ffff80001016bd14: add x24, x19, #0xc 0.00 : ffff80001016bd18: mov x2, x26 0.00 : ffff80001016bd1c: mov x1, x25 : hrtimer_interrupt(): : raw_spin_lock_irqsave(&cpu_base->lock, flags); 0.00 : ffff80001016bd20: mov x20, x0 : hrtimer_update_base(): : ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 0.00 : ffff80001016bd24: mov x3, x27 0.00 : ffff80001016bd28: mov x0, x24 0.00 : ffff80001016bd2c: bl ffff80001016ea30 : base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; 0.00 : ffff80001016bd30: ldr x2, [x19, #184] : ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 0.00 : ffff80001016bd34: mov x23, x0 : hrtimer_interrupt(): : entry_time = now = hrtimer_update_base(cpu_base); 0.00 : ffff80001016bd38: mov x28, x0 : hrtimer_update_base(): : base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; 4.08 : ffff80001016bd3c: ldr x1, [x19, #248] : base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; 0.00 : ffff80001016bd40: ldr x0, [x19, #312] 0.00 : ffff80001016bd44: mov w21, #0x3 // #3 : base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; 0.00 : ffff80001016bd48: str x2, [x19, #440] : base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; 0.00 : ffff80001016bd4c: str x1, [x19, #504] : base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; 0.00 : ffff80001016bd50: str x0, [x19, #568] : hrtimer_interrupt(): : retry: : cpu_base->in_hrtirq = 1; 0.00 : ffff80001016bd54: ldrb w0, [x19, #16] : ktime_compare(): : * cmp1 == cmp2: return 0 : * cmp1 > cmp2: return >0 : */ : static inline int ktime_compare(const ktime_t cmp1, const ktime_t cmp2) : { : if (cmp1 < cmp2) 0.00 : ffff80001016bd58: ldr x1, [x19, #48] : hrtimer_interrupt(): 0.00 : ffff80001016bd5c: orr w0, w0, #0x2 0.00 : ffff80001016bd60: strb w0, [x19, #16] : * held to prevent that a timer is enqueued in our queue via : * the migration code. This does not affect enqueueing of : * timers which run their callback and need to be requeued on : * this CPU. : */ : cpu_base->expires_next = KTIME_MAX; 4.21 : ffff80001016bd64: str x22, [x19, #32] : ktime_compare(): 0.00 : ffff80001016bd68: cmp x1, x28 0.00 : ffff80001016bd6c: b.le ffff80001016be90 : hrtimer_interrupt(): : cpu_base->softirq_expires_next = KTIME_MAX; : cpu_base->softirq_activated = 1; : raise_softirq_irqoff(HRTIMER_SOFTIRQ); : } : : __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); 0.00 : ffff80001016bd70: mov x2, x20 0.00 : ffff80001016bd74: mov w3, #0xf // #15 0.00 : ffff80001016bd78: mov x1, x28 0.00 : ffff80001016bd7c: mov x0, x19 0.00 : ffff80001016bd80: bl ffff80001016b278 <__hrtimer_run_queues> : : /* Reevaluate the clock bases for the next expiry */ : expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL); 0.00 : ffff80001016bd84: mov w1, #0xff // #255 0.00 : ffff80001016bd88: mov x0, x19 0.00 : ffff80001016bd8c: bl ffff80001016b020 <__hrtimer_get_next_event> 0.00 : ffff80001016bd90: mov x28, x0 : /* : * Store the new expiry value so the migration code can verify : * against it. : */ : cpu_base->expires_next = expires_next; : cpu_base->in_hrtirq = 0; 0.00 : ffff80001016bd94: ldrb w2, [x19, #16] : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 0.00 : ffff80001016bd98: mov x1, x20 : cpu_base->expires_next = expires_next; 2.02 : ffff80001016bd9c: str x28, [x19, #32] : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 0.00 : ffff80001016bda0: mov x0, x19 : cpu_base->in_hrtirq = 0; 0.00 : ffff80001016bda4: and w2, w2, #0xfffffffd 0.00 : ffff80001016bda8: strb w2, [x19, #16] : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 0.00 : ffff80001016bdac: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : : /* Reprogramming necessary ? */ : if (!tick_program_event(expires_next, 0)) { 0.00 : ffff80001016bdb0: mov w1, #0x0 // #0 0.00 : ffff80001016bdb4: mov x0, x28 0.00 : ffff80001016bdb8: bl ffff80001017afa8 0.00 : ffff80001016bdbc: cbz w0, ffff80001016beb0 : * overreacting on some spurious event. : * : * Acquire base lock for updating the offsets and retrieving : * the current time. : */ : raw_spin_lock_irqsave(&cpu_base->lock, flags); 0.00 : ffff80001016bdc0: mov x0, x19 0.00 : ffff80001016bdc4: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : hrtimer_update_base(): : ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 0.00 : ffff80001016bdc8: mov x3, x27 0.00 : ffff80001016bdcc: mov x2, x26 0.00 : ffff80001016bdd0: mov x1, x25 : hrtimer_interrupt(): : raw_spin_lock_irqsave(&cpu_base->lock, flags); 0.00 : ffff80001016bdd4: mov x20, x0 : hrtimer_update_base(): : ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 0.00 : ffff80001016bdd8: mov x0, x24 0.00 : ffff80001016bddc: bl ffff80001016ea30 0.00 : ffff80001016bde0: mov x28, x0 : hrtimer_interrupt(): : now = hrtimer_update_base(cpu_base); : cpu_base->nr_retries++; 0.00 : ffff80001016bde4: ldrh w0, [x19, #24] : hrtimer_update_base(): : base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; 0.00 : ffff80001016bde8: ldr x3, [x19, #184] : hrtimer_interrupt(): : if (++retries < 3) 0.00 : ffff80001016bdec: subs w21, w21, #0x1 : hrtimer_update_base(): : base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; 0.00 : ffff80001016bdf0: ldr x2, [x19, #248] : hrtimer_interrupt(): : cpu_base->nr_retries++; 0.00 : ffff80001016bdf4: add w0, w0, #0x1 : hrtimer_update_base(): : base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; 0.00 : ffff80001016bdf8: ldr x1, [x19, #312] : hrtimer_interrupt(): : cpu_base->nr_retries++; 0.00 : ffff80001016bdfc: strh w0, [x19, #24] : hrtimer_update_base(): : base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; 0.00 : ffff80001016be00: str x3, [x19, #440] : base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; 0.00 : ffff80001016be04: str x2, [x19, #504] : base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; 0.00 : ffff80001016be08: str x1, [x19, #568] : hrtimer_interrupt(): : if (++retries < 3) 0.00 : ffff80001016be0c: b.ne ffff80001016bd54 // b.any : * here. We stored the entry time, so we know exactly how long : * we spent here. We schedule the next event this amount of : * time away. : */ : cpu_base->nr_hangs++; : cpu_base->hang_detected = 1; 0.00 : ffff80001016be10: ldrb w3, [x19, #16] : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 0.00 : ffff80001016be14: mov x0, x19 : cpu_base->nr_hangs++; 0.00 : ffff80001016be18: ldrh w2, [x19, #26] : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 0.00 : ffff80001016be1c: mov x1, x20 : cpu_base->hang_detected = 1; 0.00 : ffff80001016be20: orr w3, w3, #0x4 0.00 : ffff80001016be24: strb w3, [x19, #16] : cpu_base->nr_hangs++; 0.00 : ffff80001016be28: add w2, w2, #0x1 0.00 : ffff80001016be2c: strh w2, [x19, #26] : : delta = ktime_sub(now, entry_time); 0.00 : ffff80001016be30: sub x23, x28, x23 : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 0.00 : ffff80001016be34: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : if ((unsigned int)delta > cpu_base->max_hang_time) 0.00 : ffff80001016be38: ldr w0, [x19, #28] 0.00 : ffff80001016be3c: cmp w23, w0 0.00 : ffff80001016be40: b.hi ffff80001016bea8 // b.pmore : cpu_base->max_hang_time = (unsigned int) delta; : /* : * Limit it to a sensible value as we enforce a longer : * delay. Give the CPU at least 100ms to catch up. : */ : if (delta > 100 * NSEC_PER_MSEC) 0.00 : ffff80001016be44: mov x0, #0xe100 // #57600 : expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC); : else : expires_next = ktime_add(now, delta); : tick_program_event(expires_next, 1); 0.00 : ffff80001016be48: mov w1, #0x1 // #1 : if (delta > 100 * NSEC_PER_MSEC) 0.00 : ffff80001016be4c: movk x0, #0x5f5, lsl #16 : expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC); 0.00 : ffff80001016be50: cmp x23, x0 0.00 : ffff80001016be54: add x0, x28, x0 0.00 : ffff80001016be58: add x28, x23, x28 : tick_program_event(expires_next, 1); 0.00 : ffff80001016be5c: csel x0, x28, x0, le 0.00 : ffff80001016be60: bl ffff80001017afa8 : pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta)); 0.00 : ffff80001016be64: adrp x1, ffff800011ad8000 0.00 : ffff80001016be68: ldrb w0, [x1, #1226] 0.00 : ffff80001016be6c: cbz w0, ffff80001016bee0 0.00 : ffff80001016be70: ldp x20, x21, [x29, #24] 0.00 : ffff80001016be74: ldp x22, x23, [x29, #40] 0.00 : ffff80001016be78: ldp x24, x25, [x29, #56] 0.00 : ffff80001016be7c: ldp x26, x27, [x29, #72] 0.00 : ffff80001016be80: ldr x28, [x29, #88] : } 0.00 : ffff80001016be84: ldr x19, [sp, #16] 0.00 : ffff80001016be88: ldp x29, x30, [sp], #96 0.00 : ffff80001016be8c: ret : cpu_base->softirq_activated = 1; 0.00 : ffff80001016be90: orr w0, w0, #0x8 0.00 : ffff80001016be94: strb w0, [x19, #16] : cpu_base->softirq_expires_next = KTIME_MAX; 0.00 : ffff80001016be98: str x22, [x19, #48] : raise_softirq_irqoff(HRTIMER_SOFTIRQ); 0.00 : ffff80001016be9c: mov w0, #0x8 // #8 0.00 : ffff80001016bea0: bl ffff8000100ecd98 0.00 : ffff80001016bea4: b ffff80001016bd70 : cpu_base->max_hang_time = (unsigned int) delta; 0.00 : ffff80001016bea8: str w23, [x19, #28] 0.00 : ffff80001016beac: b ffff80001016be44 : cpu_base->hang_detected = 0; 0.00 : ffff80001016beb0: ldrb w0, [x19, #16] 0.00 : ffff80001016beb4: and w0, w0, #0xfffffffb 2.06 : ffff80001016beb8: strb w0, [x19, #16] 0.00 : ffff80001016bebc: ldp x20, x21, [x29, #24] 4.01 : ffff80001016bec0: ldp x22, x23, [x29, #40] 10.35 : ffff80001016bec4: ldp x24, x25, [x29, #56] 2.10 : ffff80001016bec8: ldp x26, x27, [x29, #72] 0.00 : ffff80001016becc: ldr x28, [x29, #88] : } 0.00 : ffff80001016bed0: ldr x19, [sp, #16] 0.00 : ffff80001016bed4: ldp x29, x30, [sp], #96 0.00 : ffff80001016bed8: ret : BUG_ON(!cpu_base->hres_active); 0.00 : ffff80001016bedc: brk #0x800 : pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta)); 0.00 : ffff80001016bee0: mov w2, #0x1 // #1 0.00 : ffff80001016bee4: adrp x0, ffff8000111c8000 0.00 : ffff80001016bee8: strb w2, [x1, #1226] 0.00 : ffff80001016beec: add x0, x0, #0xaa8 0.00 : ffff80001016bef0: mov x1, x23 0.00 : ffff80001016bef4: bl ffff80001014a85c 0.00 : ffff80001016bef8: b ffff80001016be70 Percent | Source code & Disassembly of vmlinux for cycles (123 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102dca88 <__arm64_sys_io_submit>: : __arm64_sys_io_submit(): : * fail with -EBADF if the file descriptor specified in the first : * iocb is invalid. May fail with -EAGAIN if insufficient resources : * are available to queue any iocbs. Will return 0 if nr is 0. Will : * fail with -ENOSYS if not implemented. : */ : SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, 33.92 : ffff8000102dca88: stp x29, x30, [sp, #-128]! 0.00 : ffff8000102dca8c: mov x29, sp 1.58 : ffff8000102dca90: str x19, [sp, #16] 0.00 : ffff8000102dca94: stp x22, x23, [sp, #40] 0.00 : ffff8000102dca98: adrp x23, ffff800011909000 0.00 : ffff8000102dca9c: str x25, [sp, #64] 0.00 : ffff8000102dcaa0: add x1, x23, #0x908 0.00 : ffff8000102dcaa4: ldr x2, [x1] 26.98 : ffff8000102dcaa8: str x2, [x29, #120] 0.00 : ffff8000102dcaac: mov x2, #0x0 // #0 : __se_sys_io_submit(): 0.00 : ffff8000102dcab0: ldr x22, [x0, #16] : __arm64_sys_io_submit(): 0.00 : ffff8000102dcab4: ldp x1, x19, [x0] : __do_sys_io_submit(): : struct kioctx *ctx; : long ret = 0; : int i = 0; : struct blk_plug plug; : : if (unlikely(nr < 0)) 0.00 : ffff8000102dcab8: tbnz x19, #63, ffff8000102dcc80 <__arm64_sys_io_submit+0x1f8> 0.00 : ffff8000102dcabc: str x21, [x29, #32] : return -EINVAL; : : ctx = lookup_ioctx(ctx_id); 0.00 : ffff8000102dcac0: mov x0, x1 0.00 : ffff8000102dcac4: bl ffff8000102d95e8 0.00 : ffff8000102dcac8: mov x21, x0 : if (unlikely(!ctx)) { 0.00 : ffff8000102dcacc: cbz x0, ffff8000102dcc7c <__arm64_sys_io_submit+0x1f4> 0.00 : ffff8000102dcad0: str x20, [x29, #24] 0.00 : ffff8000102dcad4: str x26, [x29, #72] : pr_debug("EINVAL: invalid context id\n"); : return -EINVAL; : } : : if (nr > ctx->nr_events) 0.00 : ffff8000102dcad8: ldr w20, [x0, #144] 0.00 : ffff8000102dcadc: cmp x20, x19 0.00 : ffff8000102dcae0: csel x20, x20, x19, le : nr = ctx->nr_events; : : if (nr > AIO_PLUG_THRESHOLD) 0.00 : ffff8000102dcae4: cmp x20, #0x2 0.00 : ffff8000102dcae8: b.gt ffff8000102dcc6c <__arm64_sys_io_submit+0x1e4> : blk_start_plug(&plug); : for (i = 0; i < nr; i++) { 0.00 : ffff8000102dcaec: mov x25, #0x0 // #0 0.00 : ffff8000102dcaf0: mov x26, #0x0 // #0 0.00 : ffff8000102dcaf4: mov w19, #0x0 // #0 0.00 : ffff8000102dcaf8: cbz x20, ffff8000102dcb60 <__arm64_sys_io_submit+0xd8> 0.00 : ffff8000102dcafc: str x24, [x29, #56] : __arm64_sys_io_submit(): : SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, 0.81 : ffff8000102dcb00: mov x25, #0x0 // #0 0.00 : ffff8000102dcb04: mov w19, #0x0 // #0 0.00 : ffff8000102dcb08: mov x1, #0x0 // #0 : __do_sys_io_submit(): : struct iocb __user *user_iocb; : : if (unlikely(get_user(user_iocb, iocbpp + i))) { 0.00 : ffff8000102dcb0c: mov w24, #0x0 // #0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000102dcb10: mrs x0, sp_el0 : __range_ok(): : * Asynchronous I/O running in a kernel thread does not have the : * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag : * the user address before checking. : */ : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && : (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) 0.00 : ffff8000102dcb14: ldr w3, [x0, #44] : __do_sys_io_submit(): 0.00 : ffff8000102dcb18: add x1, x22, x1, lsl #3 : __range_ok(): : unsigned long ret, limit = current_thread_info()->addr_limit; 0.00 : ffff8000102dcb1c: ldr x2, [x0, #8] : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 0.00 : ffff8000102dcb20: tbnz w3, #21, ffff8000102dcb30 <__arm64_sys_io_submit+0xa8> : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.82 : ffff8000102dcb24: ldr x3, [x0] : __range_ok(): 0.00 : ffff8000102dcb28: mov x0, x1 0.00 : ffff8000102dcb2c: tbz w3, #26, ffff8000102dcb38 <__arm64_sys_io_submit+0xb0> : sign_extend64(): : * @index: 0 based bit index (0<=index<64) to sign bit : */ : static inline __s64 sign_extend64(__u64 value, int index) : { : __u8 shift = 63 - index; : return (__s64)(value << shift) >> shift; 0.00 : ffff8000102dcb30: sbfx x0, x1, #0, #56 : __range_ok(): : addr = untagged_addr(addr); 0.00 : ffff8000102dcb34: and x0, x1, x0 : : __chk_user_ptr(addr); : asm volatile( 0.00 : ffff8000102dcb38: adds x0, x0, #0x8 0.00 : ffff8000102dcb3c: csel x2, xzr, x2, hi // hi = pmore 0.00 : ffff8000102dcb40: csinv x0, x0, xzr, cc // cc = lo, ul, last 0.00 : ffff8000102dcb44: sbcs xzr, x0, x2 0.00 : ffff8000102dcb48: cset x0, ls // ls = plast : __do_sys_io_submit(): 0.00 : ffff8000102dcb4c: cbnz x0, ffff8000102dcbf0 <__arm64_sys_io_submit+0x168> : ret = -EFAULT; 0.00 : ffff8000102dcb50: mov x26, #0xfffffffffffffff2 // #-14 : : ret = io_submit_one(ctx, user_iocb, false); : if (ret) : break; : } : if (nr > AIO_PLUG_THRESHOLD) 0.00 : ffff8000102dcb54: cmp x20, #0x2 0.00 : ffff8000102dcb58: b.gt ffff8000102dcc50 <__arm64_sys_io_submit+0x1c8> 1.65 : ffff8000102dcb5c: ldr x24, [x29, #56] : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000102dcb60: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.65 : ffff8000102dcb64: ldr x0, [x21, #8] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000102dcb68: tst x0, #0x3 0.00 : ffff8000102dcb6c: b.ne ffff8000102dcc88 <__arm64_sys_io_submit+0x200> // b.any : get_current(): 0.00 : ffff8000102dcb70: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000102dcb74: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000102dcb78: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dcb7c: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102dcb80: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.81 : ffff8000102dcb84: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102dcb88: add x0, x0, x2 0.00 : ffff8000102dcb8c: ldxr x5, [x0] 9.78 : ffff8000102dcb90: add x5, x5, x3 0.00 : ffff8000102dcb94: stxr w4, x5, [x0] 0.00 : ffff8000102dcb98: cbnz w4, ffff8000102dcb8c <__arm64_sys_io_submit+0x104> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dcb9c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102dcba0: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102dcba4: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102dcba8: cbnz x0, ffff8000102dcc60 <__arm64_sys_io_submit+0x1d8> : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff8000102dcbac: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.76 : ffff8000102dcbb0: bl ffff800010160d20 <__rcu_read_unlock> : __do_sys_io_submit(): : blk_finish_plug(&plug); : : percpu_ref_put(&ctx->users); : return i ? i : ret; 1.64 : ffff8000102dcbb4: cmp w19, #0x0 0.00 : ffff8000102dcbb8: csel x25, x25, x26, ne // ne = any 0.00 : ffff8000102dcbbc: ldr x26, [x29, #72] 0.00 : ffff8000102dcbc0: ldp x20, x21, [x29, #24] : __arm64_sys_io_submit(): : SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, 0.00 : ffff8000102dcbc4: add x23, x23, #0x908 0.00 : ffff8000102dcbc8: mov x0, x25 0.82 : ffff8000102dcbcc: ldr x2, [x29, #120] 0.00 : ffff8000102dcbd0: ldr x1, [x23] 0.00 : ffff8000102dcbd4: eor x1, x2, x1 0.00 : ffff8000102dcbd8: cbnz x1, ffff8000102dccc4 <__arm64_sys_io_submit+0x23c> 0.00 : ffff8000102dcbdc: ldr x19, [sp, #16] 0.00 : ffff8000102dcbe0: ldp x22, x23, [sp, #40] 0.00 : ffff8000102dcbe4: ldr x25, [sp, #64] 0.82 : ffff8000102dcbe8: ldp x29, x30, [sp], #128 0.00 : ffff8000102dcbec: ret : sign_extend64(): 0.00 : ffff8000102dcbf0: sbfx x0, x1, #0, #56 : get_current(): 0.00 : ffff8000102dcbf4: mrs x2, sp_el0 : __uaccess_mask_ptr(): : asm volatile( : " bics xzr, %3, %2\n" : " csel %0, %1, xzr, eq\n" : : "=&r" (safe_ptr) : : "r" (ptr), "r" (current_thread_info()->addr_limit), : "r" (untagged_addr(ptr)) 0.00 : ffff8000102dcbf8: and x0, x1, x0 : asm volatile( 0.00 : ffff8000102dcbfc: ldr x3, [x2, #8] 0.00 : ffff8000102dcc00: bics xzr, x0, x3 0.00 : ffff8000102dcc04: csel x2, x1, xzr, eq // eq = none : : "cc"); : : csdb(); 0.00 : ffff8000102dcc08: csdb : uaccess_enable_not_uao(): : __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102dcc0c: nop : __do_sys_io_submit(): : if (unlikely(get_user(user_iocb, iocbpp + i))) { 0.82 : ffff8000102dcc10: mov w0, w24 0.00 : ffff8000102dcc14: ldr x1, [x2] : uaccess_disable_not_uao(): : __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); 0.00 : ffff8000102dcc18: nop : __do_sys_io_submit(): 0.00 : ffff8000102dcc1c: cbnz w0, ffff8000102dcb50 <__arm64_sys_io_submit+0xc8> : ret = io_submit_one(ctx, user_iocb, false); 0.00 : ffff8000102dcc20: mov w2, #0x0 // #0 0.00 : ffff8000102dcc24: mov x0, x21 0.00 : ffff8000102dcc28: bl ffff8000102db238 2.49 : ffff8000102dcc2c: sxtw x26, w0 : if (ret) 0.00 : ffff8000102dcc30: cbnz x26, ffff8000102dcb54 <__arm64_sys_io_submit+0xcc> : for (i = 0; i < nr; i++) { 0.00 : ffff8000102dcc34: add w19, w19, #0x1 0.00 : ffff8000102dcc38: sxtw x25, w19 0.00 : ffff8000102dcc3c: mov x1, x25 0.00 : ffff8000102dcc40: cmp x20, x25 0.00 : ffff8000102dcc44: b.gt ffff8000102dcb10 <__arm64_sys_io_submit+0x88> : if (nr > AIO_PLUG_THRESHOLD) 0.00 : ffff8000102dcc48: cmp x20, #0x2 0.00 : ffff8000102dcc4c: b.le ffff8000102dcb5c <__arm64_sys_io_submit+0xd4> : blk_finish_plug(&plug); 0.00 : ffff8000102dcc50: add x0, x29, #0x50 0.00 : ffff8000102dcc54: bl ffff80001045cd08 0.00 : ffff8000102dcc58: ldr x24, [x29, #56] 0.00 : ffff8000102dcc5c: b ffff8000102dcb60 <__arm64_sys_io_submit+0xd8> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102dcc60: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102dcc64: cbz x0, ffff8000102dcbac <__arm64_sys_io_submit+0x124> 14.66 : ffff8000102dcc68: b ffff8000102dcbb0 <__arm64_sys_io_submit+0x128> : __do_sys_io_submit(): : blk_start_plug(&plug); 0.00 : ffff8000102dcc6c: add x0, x29, #0x50 0.00 : ffff8000102dcc70: str x24, [x29, #56] 0.00 : ffff8000102dcc74: bl ffff800010459f98 0.00 : ffff8000102dcc78: b ffff8000102dcb00 <__arm64_sys_io_submit+0x78> 0.00 : ffff8000102dcc7c: ldr x21, [x29, #32] : return -EINVAL; 0.00 : ffff8000102dcc80: mov x25, #0xffffffffffffffea // #-22 0.00 : ffff8000102dcc84: b ffff8000102dcbc4 <__arm64_sys_io_submit+0x13c> : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102dcc88: b ffff8000102dccb4 <__arm64_sys_io_submit+0x22c> 0.00 : ffff8000102dcc8c: b ffff8000102dccb4 <__arm64_sys_io_submit+0x22c> : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000102dcc90: mov x0, #0x1 // #1 0.00 : ffff8000102dcc94: neg x0, x0 0.00 : ffff8000102dcc98: ldaddal x0, x1, [x21] 0.00 : ffff8000102dcc9c: add x0, x0, x1 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000102dcca0: cbnz x0, ffff8000102dcbb0 <__arm64_sys_io_submit+0x128> : ref->release(ref); 0.00 : ffff8000102dcca4: ldr x1, [x21, #16] 0.00 : ffff8000102dcca8: mov x0, x21 0.00 : ffff8000102dccac: blr x1 0.00 : ffff8000102dccb0: b ffff8000102dcbb0 <__arm64_sys_io_submit+0x128> : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff8000102dccb4: mov x1, #0x1 // #1 0.00 : ffff8000102dccb8: b ffff8000102ddb8c <__arm64_compat_sys_io_pgetevents_time64+0x51c> : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000102dccbc: cbnz x0, ffff8000102dcbb0 <__arm64_sys_io_submit+0x128> 0.00 : ffff8000102dccc0: b ffff8000102dcca4 <__arm64_sys_io_submit+0x21c> 0.00 : ffff8000102dccc4: stp x20, x21, [x29, #24] 0.00 : ffff8000102dccc8: str x24, [x29, #56] 0.00 : ffff8000102dcccc: str x26, [x29, #72] : __arm64_sys_io_submit(): : SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, 0.00 : ffff8000102dccd0: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (69 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010126d70 : : rebalance_domains(): : * and initiates a balancing operation if so. : * : * Balancing parameters are set up in init_sched_domains. : */ : static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) : { 0.00 : ffff800010126d70: stp x29, x30, [sp, #-192]! 0.00 : ffff800010126d74: mov x29, sp 0.00 : ffff800010126d78: stp x19, x20, [sp, #16] 0.00 : ffff800010126d7c: adrp x20, ffff800011909000 0.00 : ffff800010126d80: str x21, [sp, #32] 0.00 : ffff800010126d84: add x2, x20, #0x908 0.00 : ffff800010126d88: str x23, [sp, #48] 0.00 : ffff800010126d8c: mov x21, x0 0.00 : ffff800010126d90: stp x26, x27, [sp, #72] 0.00 : ffff800010126d94: mov w23, w1 0.00 : ffff800010126d98: ldr x0, [x2] 0.00 : ffff800010126d9c: str x0, [x29, #184] 0.00 : ffff800010126da0: mov x0, #0x0 // #0 : int continue_balancing = 1; 0.00 : ffff800010126da4: mov w0, #0x1 // #1 0.00 : ffff800010126da8: str w0, [x29, #180] : int cpu = rq->cpu; 0.00 : ffff800010126dac: ldr w0, [x21, #2568] 0.00 : ffff800010126db0: str w0, [x29, #144] : int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 0.00 : ffff800010126db4: cbz w1, ffff800010126f9c : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff800010126db8: sxtw x2, w0 0.00 : ffff800010126dbc: adrp x1, ffff800011909000 0.00 : ffff800010126dc0: str x1, [x29, #168] 0.00 : ffff800010126dc4: add x1, x1, #0x928 0.00 : ffff800010126dc8: adrp x0, ffff800011528000 0.00 : ffff800010126dcc: stp x0, x2, [x29, #152] 0.00 : ffff800010126dd0: add x0, x0, #0xe80 0.00 : ffff800010126dd4: ldr x1, [x1, x2, lsl #3] 0.00 : ffff800010126dd8: add x0, x0, x1 0.00 : ffff800010126ddc: ldr w1, [x0, #4] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff800010126de0: ldr w0, [x0, #160] 0.00 : ffff800010126de4: cmp w1, #0x0 0.00 : ffff800010126de8: ccmp w0, w1, #0x0, ne // ne = any : rebalance_domains(): : int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 0.00 : ffff800010126dec: cset w0, ne // ne = any 0.00 : ffff800010126df0: str w0, [x29, #148] : unsigned long interval; : struct sched_domain *sd; : /* Earliest time when we have to do rebalance again */ : unsigned long next_balance = jiffies + 60*HZ; 0.00 : ffff800010126df4: adrp x27, ffff800011907000 0.00 : ffff800010126df8: mov x0, #0x3a98 // #15000 0.00 : ffff800010126dfc: ldr x19, [x27, #2432] 0.00 : ffff800010126e00: add x19, x19, x0 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff800010126e04: bl ffff80001015c1b0 <__rcu_read_lock> : rebalance_domains(): : int update_next_balance = 0; : int need_serialize, need_decay = 0; : u64 max_cost = 0; : : rcu_read_lock(); : for_each_domain(cpu, sd) { 0.00 : ffff800010126e08: ldp x2, x0, [x29, #160] 0.00 : ffff800010126e0c: add x1, x0, #0x928 6.10 : ffff800010126e10: ldr x0, [x29, #152] 0.00 : ffff800010126e14: add x0, x0, #0xe80 0.00 : ffff800010126e18: ldr x1, [x1, x2, lsl #3] 0.00 : ffff800010126e1c: add x0, x0, x1 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010126e20: ldr x26, [x0, #2472] : rebalance_domains(): 0.00 : ffff800010126e24: cbz x26, ffff8000101270f4 : spin_trylock(): : raw_spin_lock_bh(&lock->rlock); : } : : static __always_inline int spin_trylock(spinlock_t *lock) : { : return raw_spin_trylock(&lock->rlock); 1.56 : ffff800010126e28: adrp x0, ffff800011afb000 0.00 : ffff800010126e2c: add x0, x0, #0xfc0 0.00 : ffff800010126e30: add x0, x0, #0x38 0.00 : ffff800010126e34: str x22, [x29, #40] 0.00 : ffff800010126e38: stp x24, x25, [x29, #56] : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff800010126e3c: adrp x22, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010126e40: str x28, [x29, #88] : rebalance_domains(): : int need_serialize, need_decay = 0; 0.00 : ffff800010126e44: mov w25, #0x0 // #0 : spin_trylock(): 0.00 : ffff800010126e48: str x0, [x29, #120] : rebalance_domains(): : int update_next_balance = 0; 0.00 : ffff800010126e4c: mov w24, #0x0 // #0 : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff800010126e50: add x0, x22, #0x240 : rebalance_domains(): : u64 max_cost = 0; 0.00 : ffff800010126e54: mov x28, #0x0 // #0 : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff800010126e58: str x0, [x29, #136] : rebalance_domains(): : /* : * Decay the newidle max times here because this is a regular : * visit to all the domains. Decay ~1% per second. : */ : if (time_after(jiffies, sd->next_decay_max_lb_cost)) { 0.00 : ffff800010126e5c: ldr x1, [x27, #2432] 0.00 : ffff800010126e60: ldp x6, x0, [x26, #80] 0.00 : ffff800010126e64: sub x0, x0, x1 1.55 : ffff800010126e68: tbz x0, #63, ffff800010126e8c : sd->max_newidle_lb_cost = : (sd->max_newidle_lb_cost * 253) / 256; 2.71 : ffff800010126e6c: lsl x0, x6, #6 : sd->next_decay_max_lb_cost = jiffies + HZ; 0.00 : ffff800010126e70: ldr x1, [x27, #2432] : (sd->max_newidle_lb_cost * 253) / 256; 0.00 : ffff800010126e74: sub x0, x0, x6 : need_decay = 1; 0.00 : ffff800010126e78: mov w25, #0x1 // #1 : sd->next_decay_max_lb_cost = jiffies + HZ; 0.00 : ffff800010126e7c: add x1, x1, #0xfa : (sd->max_newidle_lb_cost * 253) / 256; 0.00 : ffff800010126e80: add x6, x6, x0, lsl #2 0.00 : ffff800010126e84: lsr x6, x6, #8 : sd->next_decay_max_lb_cost = jiffies + HZ; 0.00 : ffff800010126e88: stp x6, x1, [x26, #80] : } : max_cost += sd->max_newidle_lb_cost; : : if (!(sd->flags & SD_LOAD_BALANCE)) 31.38 : ffff800010126e8c: ldr w0, [x26, #56] 1.56 : ffff800010126e90: and w22, w0, #0x1 0.00 : ffff800010126e94: tbz w0, #0, ffff800010126f74 : /* : * Stop the load balance at this level. There is another : * CPU in our sched group which is doing load balancing more : * actively. : */ : if (!continue_balancing) { 13.32 : ffff800010126e98: ldr w0, [x29, #180] 0.00 : ffff800010126e9c: cbnz w0, ffff800010126f00 : if (need_decay) 0.00 : ffff800010126ea0: cbnz w25, ffff800010126f74 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 1.13 : ffff800010126ea4: bl ffff800010160d20 <__rcu_read_unlock> : rebalance_domains(): : /* : * next_balance will be updated only when there is a need. : * When the cpu is attached to null domain for ex, it will not be : * updated. : */ : if (likely(update_next_balance)) { 0.00 : ffff800010126ea8: cbz w24, ffff800010126ec8 : rq->next_balance = next_balance; 0.00 : ffff800010126eac: str x19, [x21, #2376] : * nohz_idle_balance() and nohz.next_balance has been : * updated accordingly. This CPU is now running the idle load : * balance for itself and we need to update the : * nohz.next_balance accordingly. : */ : if ((idle == CPU_IDLE) && time_after(nohz.next_balance, rq->next_balance)) 0.00 : ffff800010126eb0: cbnz w23, ffff800010126ec8 0.00 : ffff800010126eb4: adrp x0, ffff800011afb000 0.00 : ffff800010126eb8: add x0, x0, #0xfc0 0.00 : ffff800010126ebc: ldr x1, [x0, #40] 0.00 : ffff800010126ec0: sub x1, x19, x1 0.00 : ffff800010126ec4: tbnz x1, #63, ffff800010127084 1.58 : ffff800010126ec8: ldr x22, [x29, #40] 1.58 : ffff800010126ecc: ldp x24, x25, [x29, #56] 0.00 : ffff800010126ed0: ldr x28, [x29, #88] : nohz.next_balance = rq->next_balance; : #endif : } : } 0.00 : ffff800010126ed4: add x20, x20, #0x908 0.00 : ffff800010126ed8: ldr x1, [x29, #184] 0.00 : ffff800010126edc: ldr x0, [x20] 0.00 : ffff800010126ee0: eor x0, x1, x0 0.00 : ffff800010126ee4: cbnz x0, ffff800010127120 0.00 : ffff800010126ee8: ldp x19, x20, [sp, #16] 0.00 : ffff800010126eec: ldr x21, [sp, #32] 0.00 : ffff800010126ef0: ldr x23, [sp, #48] 0.00 : ffff800010126ef4: ldp x26, x27, [sp, #72] 2.56 : ffff800010126ef8: ldp x29, x30, [sp], #192 0.00 : ffff800010126efc: ret : get_sd_balance_interval(): : interval *= sd->busy_factor; 0.00 : ffff800010126f00: ldr w1, [x29, #148] : rebalance_domains(): : interval = get_sd_balance_interval(sd, busy); 1.51 : ffff800010126f04: ldr w0, [x26, #40] : get_sd_balance_interval(): : interval *= sd->busy_factor; 0.00 : ffff800010126f08: cmp w1, #0x0 : rebalance_domains(): : interval = get_sd_balance_interval(sd, busy); 0.00 : ffff800010126f0c: ldr w1, [x26, #72] 0.00 : ffff800010126f10: str x6, [x29, #128] : get_sd_balance_interval(): : interval *= sd->busy_factor; 0.00 : ffff800010126f14: umull x0, w1, w0 0.00 : ffff800010126f18: mov w1, w1 : msecs_to_jiffies(): : if (__builtin_constant_p(m)) { : if ((int)m < 0) : return MAX_JIFFY_OFFSET; : return _msecs_to_jiffies(m); : } else { : return __msecs_to_jiffies(m); 0.00 : ffff800010126f1c: csel x0, x1, x0, eq // eq = none 0.00 : ffff800010126f20: bl ffff800010167188 <__msecs_to_jiffies> : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff800010126f24: cmp x0, #0x0 0.00 : ffff800010126f28: csinc x1, x0, xzr, ne // ne = any 1.58 : ffff800010126f2c: ldp x6, x0, [x29, #128] 3.22 : ffff800010126f30: ldr x0, [x0, #8] 0.00 : ffff800010126f34: cmp x1, x0 0.00 : ffff800010126f38: csel x1, x1, x0, ls // ls = plast : rebalance_domains(): : need_serialize = sd->flags & SD_SERIALIZE; 4.71 : ffff800010126f3c: ldr w0, [x26, #56] : if (need_serialize) { 0.00 : ffff800010126f40: and w2, w0, #0x400 0.00 : ffff800010126f44: str w2, [x29, #112] 0.00 : ffff800010126f48: tbnz w0, #10, ffff800010126fbc : if (time_after_eq(jiffies, sd->last_balance + interval)) { 0.00 : ffff800010126f4c: ldr x2, [x27, #2432] 0.00 : ffff800010126f50: ldr x0, [x26, #64] 0.00 : ffff800010126f54: sub x2, x2, x0 0.00 : ffff800010126f58: sub x2, x2, x1 0.00 : ffff800010126f5c: tbz x2, #63, ffff800010126fec : if (time_after(next_balance, sd->last_balance + interval)) { 1.44 : ffff800010126f60: add x1, x1, x0 : next_balance = sd->last_balance + interval; 0.00 : ffff800010126f64: subs x0, x1, x19 0.00 : ffff800010126f68: csel w24, w22, w24, mi // mi = first 0.00 : ffff800010126f6c: cmp x0, #0x0 4.40 : ffff800010126f70: csel x19, x1, x19, lt // lt = tstop : for_each_domain(cpu, sd) { 0.00 : ffff800010126f74: ldr x26, [x26] : max_cost += sd->max_newidle_lb_cost; 0.00 : ffff800010126f78: add x28, x28, x6 : for_each_domain(cpu, sd) { 0.00 : ffff800010126f7c: cbnz x26, ffff800010126e5c : if (need_decay) { 0.00 : ffff800010126f80: cbz w25, ffff800010126ea4 : max((u64)sysctl_sched_migration_cost, max_cost); 0.00 : ffff800010126f84: mov x0, #0xa120 // #41248 0.00 : ffff800010126f88: movk x0, #0x7, lsl #16 0.00 : ffff800010126f8c: cmp x28, x0 0.00 : ffff800010126f90: csel x7, x28, x0, cs // cs = hs, nlast : rq->max_idle_balance_cost = 0.00 : ffff800010126f94: str x7, [x21, #2832] 0.00 : ffff800010126f98: b ffff800010126ea4 1.46 : ffff800010126f9c: ldrsw x0, [x29, #144] 0.00 : ffff800010126fa0: str x0, [x29, #160] 0.00 : ffff800010126fa4: adrp x0, ffff800011528000 : int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 0.00 : ffff800010126fa8: str wzr, [x29, #148] 0.00 : ffff800010126fac: str x0, [x29, #152] 0.00 : ffff800010126fb0: adrp x0, ffff800011909000 0.00 : ffff800010126fb4: str x0, [x29, #168] 0.00 : ffff800010126fb8: b ffff800010126df4 : spin_trylock(): 0.00 : ffff800010126fbc: ldr x0, [x29, #120] 1.51 : ffff800010126fc0: str x6, [x29, #104] 0.00 : ffff800010126fc4: str x1, [x29, #128] 0.00 : ffff800010126fc8: bl ffff800010cd7fe0 <_raw_spin_trylock> : rebalance_domains(): : if (!spin_trylock(&balancing)) 1.56 : ffff800010126fcc: ldr x6, [x29, #104] 0.00 : ffff800010126fd0: ldr x1, [x29, #128] 0.00 : ffff800010126fd4: cbz w0, ffff8000101270ec : if (time_after_eq(jiffies, sd->last_balance + interval)) { 0.00 : ffff800010126fd8: ldr x0, [x27, #2432] 1.57 : ffff800010126fdc: ldr x2, [x26, #64] 0.00 : ffff800010126fe0: sub x0, x0, x2 0.00 : ffff800010126fe4: sub x0, x0, x1 0.00 : ffff800010126fe8: tbnz x0, #63, ffff80001012705c : if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { 1.65 : ffff800010126fec: ldr w0, [x29, #144] 0.00 : ffff800010126ff0: add x4, x29, #0xb4 2.97 : ffff800010126ff4: str x6, [x29, #128] 0.00 : ffff800010126ff8: mov w3, w23 0.00 : ffff800010126ffc: mov x2, x26 0.00 : ffff800010127000: mov x1, x21 0.00 : ffff800010127004: bl ffff8000101261a8 0.00 : ffff800010127008: ldr x6, [x29, #128] 0.00 : ffff80001012700c: cbnz w0, ffff800010127098 : sd->last_balance = jiffies; 0.00 : ffff800010127010: ldr x0, [x27, #2432] : get_sd_balance_interval(): : if (cpu_busy) 0.00 : ffff800010127014: ldr w1, [x29, #148] : rebalance_domains(): : sd->last_balance = jiffies; 0.00 : ffff800010127018: str x0, [x26, #64] : get_sd_balance_interval(): : unsigned long interval = sd->balance_interval; 0.00 : ffff80001012701c: ldr w0, [x26, #72] : if (cpu_busy) 0.00 : ffff800010127020: cbz w1, ffff800010127034 : rebalance_domains(): : interval = get_sd_balance_interval(sd, busy); 0.00 : ffff800010127024: ldr w1, [x26, #40] : get_sd_balance_interval(): : interval *= sd->busy_factor; 0.00 : ffff800010127028: mov w1, w1 0.00 : ffff80001012702c: str w22, [x29, #148] 0.00 : ffff800010127030: mul x0, x0, x1 0.00 : ffff800010127034: str x6, [x29, #128] : msecs_to_jiffies(): 0.00 : ffff800010127038: bl ffff800010167188 <__msecs_to_jiffies> : get_sd_balance_interval(): : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff80001012703c: cmp x0, #0x0 0.00 : ffff800010127040: csinc x1, x0, xzr, ne // ne = any 0.00 : ffff800010127044: ldp x6, x0, [x29, #128] 0.00 : ffff800010127048: ldr x0, [x0, #8] 0.00 : ffff80001012704c: cmp x1, x0 0.00 : ffff800010127050: csel x1, x1, x0, ls // ls = plast : rebalance_domains(): : if (need_serialize) 0.00 : ffff800010127054: ldr w0, [x29, #112] 0.00 : ffff800010127058: cbz w0, ffff8000101270ec : spin_unlock(): : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ : } while (0) : : static __always_inline void spin_unlock(spinlock_t *lock) : { : raw_spin_unlock(&lock->rlock); 3.12 : ffff80001012705c: adrp x0, ffff800011afb000 0.00 : ffff800010127060: add x0, x0, #0xfc0 0.00 : ffff800010127064: add x0, x0, #0x38 0.00 : ffff800010127068: str x6, [x29, #112] 1.56 : ffff80001012706c: str x1, [x29, #128] 0.00 : ffff800010127070: bl ffff800010cd7c38 <_raw_spin_unlock> 0.00 : ffff800010127074: ldr x0, [x26, #64] 1.40 : ffff800010127078: ldr x6, [x29, #112] 1.30 : ffff80001012707c: ldr x1, [x29, #128] 0.00 : ffff800010127080: b ffff800010126f60 : rebalance_domains(): : nohz.next_balance = rq->next_balance; 0.00 : ffff800010127084: str x19, [x0, #40] : } 0.00 : ffff800010127088: ldr x22, [x29, #40] 0.00 : ffff80001012708c: ldp x24, x25, [x29, #56] 0.00 : ffff800010127090: ldr x28, [x29, #88] 0.00 : ffff800010127094: b ffff800010126ed4 : idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 0.00 : ffff800010127098: ldr w0, [x29, #144] 0.00 : ffff80001012709c: bl ffff800010118820 0.00 : ffff8000101270a0: str w0, [x29, #148] : busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 0.00 : ffff8000101270a4: ldr x6, [x29, #128] 0.00 : ffff8000101270a8: cbnz w0, ffff8000101270fc : sched_idle_cpu(): : return sched_idle_rq(cpu_rq(cpu)); 0.00 : ffff8000101270ac: ldr x1, [x29, #168] 0.00 : ffff8000101270b0: ldp x0, x2, [x29, #152] 0.00 : ffff8000101270b4: add x1, x1, #0x928 0.00 : ffff8000101270b8: add x0, x0, #0xe80 0.00 : ffff8000101270bc: ldr x1, [x1, x2, lsl #3] 0.00 : ffff8000101270c0: add x1, x0, x1 0.00 : ffff8000101270c4: ldr w0, [x26, #72] 0.00 : ffff8000101270c8: ldr w2, [x1, #4] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff8000101270cc: ldr w1, [x1, #160] 0.00 : ffff8000101270d0: cmp w2, w1 : rebalance_domains(): : sd->last_balance = jiffies; 0.00 : ffff8000101270d4: ldr x1, [x27, #2432] 0.00 : ffff8000101270d8: str x1, [x26, #64] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff8000101270dc: b.eq ffff800010127114 // b.none : rebalance_domains(): : idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 0.00 : ffff8000101270e0: mov w23, #0x1 // #1 : interval = get_sd_balance_interval(sd, busy); 0.00 : ffff8000101270e4: ldr w1, [x26, #40] 0.00 : ffff8000101270e8: b ffff800010127028 0.00 : ffff8000101270ec: ldr x0, [x26, #64] 0.00 : ffff8000101270f0: b ffff800010126f60 : rcu_read_unlock(): 0.00 : ffff8000101270f4: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff8000101270f8: b ffff800010126ed4 : rebalance_domains(): : sd->last_balance = jiffies; 0.00 : ffff8000101270fc: ldr x1, [x27, #2432] : idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 0.00 : ffff800010127100: mov w23, #0x0 // #0 : get_sd_balance_interval(): : unsigned long interval = sd->balance_interval; 0.00 : ffff800010127104: ldr w0, [x26, #72] : rebalance_domains(): : sd->last_balance = jiffies; 0.00 : ffff800010127108: str x1, [x26, #64] : busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 0.00 : ffff80001012710c: str wzr, [x29, #148] 0.00 : ffff800010127110: b ffff800010127034 : idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 0.00 : ffff800010127114: mov w23, #0x1 // #1 : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff800010127118: cbz w2, ffff8000101270e0 0.00 : ffff80001012711c: b ffff800010127034 0.00 : ffff800010127120: str x22, [x29, #40] 0.00 : ffff800010127124: stp x24, x25, [x29, #56] 0.00 : ffff800010127128: str x28, [x29, #88] : rebalance_domains(): : } 0.00 : ffff80001012712c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (105 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001046ce80 : : blk_mq_sched_insert_requests(): : } : : void blk_mq_sched_insert_requests(struct blk_mq_hw_ctx *hctx, : struct blk_mq_ctx *ctx, : struct list_head *list, bool run_queue_async) : { 0.00 : ffff80001046ce80: stp x29, x30, [sp, #-64]! 0.00 : ffff80001046ce84: mov x29, sp 0.00 : ffff80001046ce88: stp x19, x20, [sp, #16] 0.00 : ffff80001046ce8c: mov x19, x0 3.87 : ffff80001046ce90: stp x21, x22, [sp, #32] 0.00 : ffff80001046ce94: mov x21, x2 0.00 : ffff80001046ce98: stp x23, x24, [sp, #48] 0.00 : ffff80001046ce9c: and w22, w3, #0xff 0.00 : ffff80001046cea0: mov x24, x1 : struct elevator_queue *e; : struct request_queue *q = hctx->queue; 0.97 : ffff80001046cea4: ldr x20, [x0, #208] : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001046cea8: bl ffff80001015c1b0 <__rcu_read_lock> : blk_mq_sched_insert_requests(): : /* : * blk_mq_sched_insert_requests() is called from flush plug : * context only, and hold one usage counter to prevent queue : * from being released. : */ : percpu_ref_get(&q->q_usage_counter); 0.00 : ffff80001046ceac: add x23, x20, #0x5c0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.79 : ffff80001046ceb0: ldr x0, [x20, #1480] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001046ceb4: tst x0, #0x3 0.00 : ffff80001046ceb8: b.ne ffff80001046d010 // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 2.80 : ffff80001046cebc: mrs x1, sp_el0 : __read_once_size(): 1.86 : ffff80001046cec0: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001046cec4: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001046cec8: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff80001046cecc: mov x3, #0x1 // #1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.90 : ffff80001046ced0: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff80001046ced4: add x0, x0, x2 0.97 : ffff80001046ced8: ldxr x5, [x0] 20.16 : ffff80001046cedc: add x5, x5, x3 0.00 : ffff80001046cee0: stxr w4, x5, [x0] 0.00 : ffff80001046cee4: cbnz w4, ffff80001046ced8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001046cee8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001046ceec: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 2.91 : ffff80001046cef0: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001046cef4: cbnz x0, ffff80001046cfbc : percpu_ref_get_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_add(*percpu_count, nr); 0.00 : ffff80001046cef8: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff80001046cefc: bl ffff800010160d20 <__rcu_read_unlock> : blk_mq_sched_insert_requests(): : : e = hctx->queue->elevator; 0.00 : ffff80001046cf00: ldr x0, [x19, #208] 0.00 : ffff80001046cf04: ldr x0, [x0, #8] : if (e && e->type->ops.insert_requests) 0.00 : ffff80001046cf08: cbz x0, ffff80001046cfd4 0.00 : ffff80001046cf0c: ldr x0, [x0] 0.00 : ffff80001046cf10: ldr x3, [x0, #112] 0.00 : ffff80001046cf14: cbz x3, ffff80001046cfe0 : e->type->ops.insert_requests(hctx, list, false); 0.00 : ffff80001046cf18: mov w2, #0x0 // #0 0.00 : ffff80001046cf1c: mov x1, x21 0.00 : ffff80001046cf20: mov x0, x19 0.00 : ffff80001046cf24: blr x3 : goto out; : } : blk_mq_insert_requests(hctx, ctx, list); : } : : blk_mq_run_hw_queue(hctx, run_queue_async); 0.00 : ffff80001046cf28: mov w1, w22 0.00 : ffff80001046cf2c: mov x0, x19 0.00 : ffff80001046cf30: bl ffff800010465418 : rcu_read_lock(): : __rcu_read_lock(); 1.91 : ffff80001046cf34: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : __READ_ONCE_SIZE; 0.96 : ffff80001046cf38: ldr x0, [x20, #1480] : __ref_is_percpu(): : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001046cf3c: tst x0, #0x3 0.00 : ffff80001046cf40: b.ne ffff80001046d030 // b.any : get_current(): 0.00 : ffff80001046cf44: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff80001046cf48: ldr w2, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff80001046cf4c: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.93 : ffff80001046cf50: str w2, [x1, #16] : __percpu_add_case_64(): 0.00 : ffff80001046cf54: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001046cf58: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff80001046cf5c: add x0, x0, x2 0.00 : ffff80001046cf60: ldxr x5, [x0] 30.37 : ffff80001046cf64: add x5, x5, x3 0.00 : ffff80001046cf68: stxr w4, x5, [x0] 0.00 : ffff80001046cf6c: cbnz w4, ffff80001046cf60 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001046cf70: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001046cf74: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001046cf78: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001046cf7c: cbnz x0, ffff80001046cf9c : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff80001046cf80: bl ffff800010cd2e78 : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff80001046cf84: bl ffff800010160d20 <__rcu_read_unlock> : blk_mq_sched_insert_requests(): : out: : percpu_ref_put(&q->q_usage_counter); : } 0.00 : ffff80001046cf88: ldp x19, x20, [sp, #16] 0.00 : ffff80001046cf8c: ldp x21, x22, [sp, #32] 0.00 : ffff80001046cf90: ldp x23, x24, [sp, #48] 0.00 : ffff80001046cf94: ldp x29, x30, [sp], #64 0.00 : ffff80001046cf98: ret : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001046cf9c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001046cfa0: cbz x0, ffff80001046cf80 : rcu_read_unlock(): 0.95 : ffff80001046cfa4: bl ffff800010160d20 <__rcu_read_unlock> : blk_mq_sched_insert_requests(): 0.00 : ffff80001046cfa8: ldp x19, x20, [sp, #16] 0.00 : ffff80001046cfac: ldp x21, x22, [sp, #32] 1.86 : ffff80001046cfb0: ldp x23, x24, [sp, #48] 0.00 : ffff80001046cfb4: ldp x29, x30, [sp], #64 0.00 : ffff80001046cfb8: ret : __read_once_size(): 1.93 : ffff80001046cfbc: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.85 : ffff80001046cfc0: cbz x0, ffff80001046cef8 : rcu_read_unlock(): 4.79 : ffff80001046cfc4: bl ffff800010160d20 <__rcu_read_unlock> : blk_mq_sched_insert_requests(): : e = hctx->queue->elevator; 0.95 : ffff80001046cfc8: ldr x0, [x19, #208] 2.87 : ffff80001046cfcc: ldr x0, [x0, #8] : if (e && e->type->ops.insert_requests) 0.00 : ffff80001046cfd0: cbnz x0, ffff80001046cf0c : if (!hctx->dispatch_busy && !e && !run_queue_async) { 2.87 : ffff80001046cfd4: ldr w0, [x19, #264] 0.00 : ffff80001046cfd8: cbnz w0, ffff80001046cfe0 0.00 : ffff80001046cfdc: cbz w22, ffff80001046cff4 : blk_mq_insert_requests(hctx, ctx, list); 0.00 : ffff80001046cfe0: mov x2, x21 0.00 : ffff80001046cfe4: mov x1, x24 0.00 : ffff80001046cfe8: mov x0, x19 0.00 : ffff80001046cfec: bl ffff800010467d28 0.00 : ffff80001046cff0: b ffff80001046cf28 : blk_mq_try_issue_list_directly(hctx, list); 0.00 : ffff80001046cff4: mov x0, x19 0.00 : ffff80001046cff8: mov x1, x21 0.00 : ffff80001046cffc: bl ffff800010468000 : __read_once_size(): 0.00 : ffff80001046d000: ldr x0, [x21] : blk_mq_sched_insert_requests(): : if (list_empty(list)) 0.00 : ffff80001046d004: cmp x21, x0 0.00 : ffff80001046d008: b.ne ffff80001046cfe0 // b.any : out: 10.54 : ffff80001046d00c: b ffff80001046cf34 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001046d010: b ffff80001046d024 0.00 : ffff80001046d014: b ffff80001046d024 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff80001046d018: mov x0, #0x1 // #1 0.00 : ffff80001046d01c: stadd x0, [x23] 0.00 : ffff80001046d020: b ffff80001046cefc : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff80001046d024: add x2, x20, #0x5c0 0.00 : ffff80001046d028: b ffff80001046d3b4 0.00 : ffff80001046d02c: b ffff80001046cefc : arch_static_branch_jump(): 0.00 : ffff80001046d030: b ffff80001046d060 0.00 : ffff80001046d034: b ffff80001046d060 : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001046d038: mov x0, #0x1 // #1 0.00 : ffff80001046d03c: add x2, x20, #0x5c0 0.00 : ffff80001046d040: neg x0, x0 0.00 : ffff80001046d044: ldaddal x0, x1, [x2] 0.00 : ffff80001046d048: add x0, x0, x1 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001046d04c: cbnz x0, ffff80001046cf84 : ref->release(ref); 0.00 : ffff80001046d050: ldr x1, [x23, #16] 0.00 : ffff80001046d054: mov x0, x23 0.00 : ffff80001046d058: blr x1 0.00 : ffff80001046d05c: b ffff80001046cf84 : __ll_sc_atomic64_sub_return(): : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff80001046d060: mov x1, #0x1 // #1 0.00 : ffff80001046d064: add x3, x20, #0x5c0 0.00 : ffff80001046d068: b ffff80001046d3cc : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001046d06c: cbnz x0, ffff80001046cf84 0.00 : ffff80001046d070: b ffff80001046d050 Percent | Source code & Disassembly of vmlinux for cycles (94 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101d4b78 : : generic_file_read_iter(): : * * number of bytes copied, even for partial reads : * * negative error code if nothing was read : */ : ssize_t : generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) : { 1.08 : ffff8000101d4b78: stp x29, x30, [sp, #-240]! 0.00 : ffff8000101d4b7c: mov x29, sp 8.50 : ffff8000101d4b80: stp x27, x28, [sp, #80] 0.00 : ffff8000101d4b84: mov x28, x1 0.00 : ffff8000101d4b88: str x19, [sp, #16] 0.00 : ffff8000101d4b8c: adrp x1, ffff800011909000 0.00 : ffff8000101d4b90: add x1, x1, #0x908 0.00 : ffff8000101d4b94: mov x27, x0 : size_t count = iov_iter_count(iter); 1.07 : ffff8000101d4b98: ldr x19, [x28, #16] : { 1.09 : ffff8000101d4b9c: ldr x0, [x1] 1.07 : ffff8000101d4ba0: str x0, [x29, #232] 0.00 : ffff8000101d4ba4: mov x0, #0x0 // #0 : ssize_t retval = 0; : : if (!count) 0.00 : ffff8000101d4ba8: cbz x19, ffff8000101d4f00 0.00 : ffff8000101d4bac: str x20, [x29, #24] 0.00 : ffff8000101d4bb0: str x26, [x29, #72] 0.00 : ffff8000101d4bb4: ldp x0, x1, [x27] 0.00 : ffff8000101d4bb8: mov x2, x0 : goto out; /* skip atime */ : : if (iocb->ki_flags & IOCB_DIRECT) { 0.00 : ffff8000101d4bbc: ldr w0, [x27, #32] 0.00 : ffff8000101d4bc0: str x2, [x29, #160] 4.26 : ffff8000101d4bc4: ldr x26, [x2, #240] 3.13 : ffff8000101d4bc8: ldr x3, [x26] 8.27 : ffff8000101d4bcc: str x3, [x29, #144] 0.00 : ffff8000101d4bd0: tbz w0, #2, ffff8000101d4f54 0.00 : ffff8000101d4bd4: add x2, x1, x19 : struct file *file = iocb->ki_filp; : struct address_space *mapping = file->f_mapping; : struct inode *inode = mapping->host; : loff_t size; : : size = i_size_read(inode); 0.00 : ffff8000101d4bd8: ldr x20, [x3, #80] 0.00 : ffff8000101d4bdc: sub x2, x2, #0x1 : if (iocb->ki_flags & IOCB_NOWAIT) { 0.00 : ffff8000101d4be0: tbnz w0, #7, ffff8000101d4f30 : if (filemap_range_has_page(mapping, iocb->ki_pos, : iocb->ki_pos + count - 1)) : return -EAGAIN; : } else { : retval = filemap_write_and_wait_range(mapping, 5.34 : ffff8000101d4be4: mov x0, x26 0.00 : ffff8000101d4be8: bl ffff8000101d4ad8 0.00 : ffff8000101d4bec: sxtw x0, w0 0.00 : ffff8000101d4bf0: str x0, [x29, #168] : iocb->ki_pos, : iocb->ki_pos + count - 1); : if (retval < 0) 0.00 : ffff8000101d4bf4: tbnz x0, #63, ffff8000101d5374 : file_accessed(): : : extern bool atime_needs_update(const struct path *, struct inode *); : extern void touch_atime(const struct path *); : static inline void file_accessed(struct file *file) : { : if (!(file->f_flags & O_NOATIME)) 3.17 : ffff8000101d4bf8: ldr x0, [x29, #160] 3.23 : ffff8000101d4bfc: ldr w0, [x0, #64] 1.09 : ffff8000101d4c00: tbz w0, #18, ffff8000101d5428 : generic_file_read_iter(): : goto out; : } : : file_accessed(file); : : retval = mapping->a_ops->direct_IO(iocb, iter); 6.46 : ffff8000101d4c04: ldr x2, [x26, #112] 0.00 : ffff8000101d4c08: mov x1, x28 0.00 : ffff8000101d4c0c: mov x0, x27 4.33 : ffff8000101d4c10: ldr x2, [x2, #88] 0.00 : ffff8000101d4c14: blr x2 2.18 : ffff8000101d4c18: str x0, [x29, #168] 0.00 : ffff8000101d4c1c: mov x2, x0 : if (retval >= 0) { 4.34 : ffff8000101d4c20: tbnz x0, #63, ffff8000101d540c : iocb->ki_pos += retval; 0.00 : ffff8000101d4c24: ldr x1, [x27, #8] : count -= retval; 0.00 : ffff8000101d4c28: sub x19, x19, x0 : } : iov_iter_revert(iter, count - iov_iter_count(iter)); 0.00 : ffff8000101d4c2c: mov x0, x28 : iocb->ki_pos += retval; 0.00 : ffff8000101d4c30: add x1, x1, x2 0.00 : ffff8000101d4c34: str x1, [x27, #8] : iov_iter_revert(iter, count - iov_iter_count(iter)); 0.00 : ffff8000101d4c38: ldr x1, [x28, #16] 0.00 : ffff8000101d4c3c: sub x1, x19, x1 0.00 : ffff8000101d4c40: bl ffff800010488720 : * there was a short read because we hit EOF, go ahead : * and return. Otherwise fallthrough to buffered io for : * the rest of the read. Buffered reads will not work for : * DAX files, so don't bother trying. : */ : if (retval < 0 || !count || iocb->ki_pos >= size || 0.00 : ffff8000101d4c44: cbz x19, ffff8000101d5374 0.00 : ffff8000101d4c48: ldr x1, [x27, #8] 0.00 : ffff8000101d4c4c: cmp x1, x20 0.00 : ffff8000101d4c50: b.ge ffff8000101d5374 // b.tcont 0.00 : ffff8000101d4c54: ldr x0, [x27] 0.00 : ffff8000101d4c58: str x0, [x29, #160] 0.00 : ffff8000101d4c5c: ldr x26, [x0, #240] 0.00 : ffff8000101d4c60: ldr x0, [x26] 0.00 : ffff8000101d4c64: str x0, [x29, #144] : generic_file_buffered_read(): : if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) 0.00 : ffff8000101d4c68: ldr x0, [x29, #144] : struct file_ra_state *ra = &filp->f_ra; 0.00 : ffff8000101d4c6c: ldr x2, [x29, #160] : if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) 0.00 : ffff8000101d4c70: ldr x0, [x0, #40] : struct file_ra_state *ra = &filp->f_ra; 0.00 : ffff8000101d4c74: add x2, x2, #0x98 0.00 : ffff8000101d4c78: str x2, [x29, #136] : if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) 0.00 : ffff8000101d4c7c: ldr x2, [x0, #32] 0.00 : ffff8000101d4c80: cmp x2, x1 0.00 : ffff8000101d4c84: b.le ffff8000101d4ef8 0.00 : ffff8000101d4c88: stp x21, x22, [x29, #32] 0.00 : ffff8000101d4c8c: stp x23, x24, [x29, #48] 0.00 : ffff8000101d4c90: str x25, [x29, #64] : iov_iter_truncate(): : * count doesn't have to fit in size_t - comparison extends both : * operands to u64 here and any value that would be truncated by : * conversion in assignement is by definition greater than all : * values of size_t, including old i->count. : */ : if (i->count > count) 0.00 : ffff8000101d4c94: ldr x0, [x28, #16] 0.00 : ffff8000101d4c98: cmp x2, x0 0.00 : ffff8000101d4c9c: b.cs ffff8000101d4cac // b.hs, b.nlast : i->count = count; 0.00 : ffff8000101d4ca0: str x2, [x28, #16] 0.00 : ffff8000101d4ca4: mov x0, x2 0.00 : ffff8000101d4ca8: ldr x1, [x27, #8] : generic_file_buffered_read(): : prev_index = ra->prev_pos >> PAGE_SHIFT; 0.00 : ffff8000101d4cac: ldr x2, [x29, #136] : last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; 0.00 : ffff8000101d4cb0: add x0, x0, #0xfff 0.00 : ffff8000101d4cb4: add x0, x0, x1 : index = *ppos >> PAGE_SHIFT; 0.00 : ffff8000101d4cb8: asr x3, x1, #12 : INIT_LIST_HEAD(): : * Initializes the list_head to point to itself. If it is a list header, : * the result is an empty list. : */ : static inline void INIT_LIST_HEAD(struct list_head *list) : { : WRITE_ONCE(list->next, list); 0.00 : ffff8000101d4cbc: add x21, x29, #0xd8 : generic_file_buffered_read(): 0.00 : ffff8000101d4cc0: str x3, [x29, #152] : prev_index = ra->prev_pos >> PAGE_SHIFT; 0.00 : ffff8000101d4cc4: ldr x2, [x2, #24] : last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; 0.00 : ffff8000101d4cc8: lsr x0, x0, #12 0.00 : ffff8000101d4ccc: str x0, [x29, #112] : offset = *ppos & ~PAGE_MASK; 0.00 : ffff8000101d4cd0: and x0, x1, #0xfff : prev_offset = ra->prev_pos & (PAGE_SIZE-1); 0.00 : ffff8000101d4cd4: and w3, w2, #0xfff 0.00 : ffff8000101d4cd8: str w3, [x29, #120] : prev_index = ra->prev_pos >> PAGE_SHIFT; 0.00 : ffff8000101d4cdc: asr x2, x2, #12 0.00 : ffff8000101d4ce0: str x2, [x29, #104] : offset = *ppos & ~PAGE_MASK; 0.00 : ffff8000101d4ce4: str x0, [x29, #128] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000101d4ce8: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000101d4cec: ldr x1, [x0] : fatal_signal_pending(): : return unlikely(sigismember(&p->pending.signal, SIGKILL)); : } : : static inline int fatal_signal_pending(struct task_struct *p) : { : return signal_pending(p) && __fatal_signal_pending(p); 0.00 : ffff8000101d4cf0: tbnz w1, #0, ffff8000101d5438 : find_get_page(): : * Otherwise, %NULL is returned. : */ : static inline struct page *find_get_page(struct address_space *mapping, : pgoff_t offset) : { : return pagecache_get_page(mapping, offset, 0, 0); 0.00 : ffff8000101d4cf4: ldr x1, [x29, #152] 0.00 : ffff8000101d4cf8: mov w3, #0x0 // #0 0.00 : ffff8000101d4cfc: mov w2, #0x0 // #0 0.00 : ffff8000101d4d00: mov x0, x26 0.00 : ffff8000101d4d04: bl ffff8000101d3118 0.00 : ffff8000101d4d08: mov x22, x0 : generic_file_buffered_read(): : if (!page) { 0.00 : ffff8000101d4d0c: cbz x0, ffff8000101d524c : test_bit(): 0.00 : ffff8000101d4d10: ldr x0, [x22] : generic_file_buffered_read(): : if (PageReadahead(page)) { 0.00 : ffff8000101d4d14: tbnz w0, #18, ffff8000101d5228 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101d4d18: ldr x0, [x22, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff8000101d4d1c: tst x0, #0x1 0.00 : ffff8000101d4d20: sub x0, x0, #0x1 0.00 : ffff8000101d4d24: csel x0, x0, x22, ne // ne = any : test_bit(): 0.00 : ffff8000101d4d28: ldr x0, [x0] : PageUptodate(): : * We can skip the barrier if the page is not uptodate, because : * we wouldn't be reading anything from it. : * : * See SetPageUptodate() for the other side of the story. : */ : if (ret) 0.00 : ffff8000101d4d2c: tbnz w0, #2, ffff8000101d4f64 : generic_file_buffered_read(): : if (iocb->ki_flags & IOCB_NOWAIT) { 0.00 : ffff8000101d4d30: ldr w0, [x27, #32] 0.00 : ffff8000101d4d34: tbnz w0, #7, ffff8000101d5458 : __read_once_size(): 0.00 : ffff8000101d4d38: ldr x0, [x22, #8] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000101d4d3c: tst x0, #0x1 0.00 : ffff8000101d4d40: sub x0, x0, #0x1 0.00 : ffff8000101d4d44: csel x0, x0, x22, ne // ne = any : test_bit(): 0.00 : ffff8000101d4d48: ldr x0, [x0] : wait_on_page_locked_killable(): : wait_on_page_bit(compound_head(page), PG_locked); : } : : static inline int wait_on_page_locked_killable(struct page *page) : { : if (!PageLocked(page)) 0.00 : ffff8000101d4d4c: tbz w0, #0, ffff8000101d508c : __read_once_size(): 0.00 : ffff8000101d4d50: ldr x19, [x22, #8] : page_waitqueue(): : return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)]; 0.00 : ffff8000101d4d54: adrp x24, ffff800011907000 0.00 : ffff8000101d4d58: add x20, x24, #0xb40 : compound_head(): 0.00 : ffff8000101d4d5c: tst x19, #0x1 0.00 : ffff8000101d4d60: sub x19, x19, #0x1 0.00 : ffff8000101d4d64: csel x19, x19, x22, ne // ne = any : __read_once_size(): 0.00 : ffff8000101d4d68: ldr x0, [x19, #8] : compound_head(): 0.00 : ffff8000101d4d6c: tst x0, #0x1 0.00 : ffff8000101d4d70: sub x0, x0, #0x1 0.00 : ffff8000101d4d74: csel x0, x0, x19, ne // ne = any : test_bit(): 0.00 : ffff8000101d4d78: ldr x1, [x0] : hash_64_generic(): : #endif : static __always_inline u32 hash_64_generic(u64 val, unsigned int bits) : { : #if BITS_PER_LONG == 64 : /* 64x64-bit multiply is efficient on all 64-bit processors */ : return val * GOLDEN_RATIO_64 >> (64 - bits); 0.00 : ffff8000101d4d7c: mov x0, #0x83eb // #33771 0.00 : ffff8000101d4d80: movk x0, #0x80b5, lsl #16 0.00 : ffff8000101d4d84: movk x0, #0x8646, lsl #32 0.00 : ffff8000101d4d88: movk x0, #0x61c8, lsl #48 0.00 : ffff8000101d4d8c: mul x23, x19, x0 0.00 : ffff8000101d4d90: lsr x23, x23, #56 0.00 : ffff8000101d4d94: add x23, x23, x23, lsl #1 0.00 : ffff8000101d4d98: lsl x23, x23, #3 : page_waitqueue(): 0.00 : ffff8000101d4d9c: add x20, x20, x23 : PageUptodate(): : if (ret) 0.00 : ffff8000101d4da0: tbz w1, #2, ffff8000101d5300 : smp_rmb(); 0.00 : ffff8000101d4da4: dmb ishld 0.00 : ffff8000101d4da8: str wzr, [x29, #124] : __add_wait_queue_entry_tail(): : __add_wait_queue(wq_head, wq_entry); : } : : static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) : { : list_add_tail(&wq_entry->entry, &wq_head->head); 0.00 : ffff8000101d4dac: add x0, x24, #0xb40 0.00 : ffff8000101d4db0: add x23, x23, #0x8 0.00 : ffff8000101d4db4: add x23, x23, x0 : __lse_atomic64_or(): : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) 0.00 : ffff8000101d4db8: mov x25, #0x80 // #128 : get_current(): 0.00 : ffff8000101d4dbc: mrs x0, sp_el0 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101d4dc0: mov x24, #0x102 // #258 : wait_on_page_bit_common(): : wait_page.page = page; 0.00 : ffff8000101d4dc4: str x19, [x29, #176] : wait_page.bit_nr = bit_nr; 0.00 : ffff8000101d4dc8: str wzr, [x29, #184] : wait->flags = behavior == EXCLUSIVE ? WQ_FLAG_EXCLUSIVE : 0; 0.00 : ffff8000101d4dcc: str wzr, [x29, #192] : init_wait(wait); 0.00 : ffff8000101d4dd0: str x0, [x29, #200] : wait->func = wake_page_function; 0.00 : ffff8000101d4dd4: adrp x0, ffff8000101d0000 : __write_once_size(): 0.00 : ffff8000101d4dd8: str x21, [x29, #216] : wait_on_page_bit_common(): 0.00 : ffff8000101d4ddc: add x0, x0, #0x2c8 0.00 : ffff8000101d4de0: str x0, [x29, #208] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff8000101d4de4: str x21, [x29, #224] : spin_lock_irq(): : raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \ : } while (0) : : static __always_inline void spin_lock_irq(spinlock_t *lock) : { : raw_spin_lock_irq(&lock->rlock); 0.00 : ffff8000101d4de8: mov x0, x20 0.00 : ffff8000101d4dec: bl ffff800010cd8460 <_raw_spin_lock_irq> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101d4df0: ldr x0, [x29, #216] : wait_on_page_bit_common(): : if (likely(list_empty(&wait->entry))) { 0.00 : ffff8000101d4df4: cmp x21, x0 0.00 : ffff8000101d4df8: b.ne ffff8000101d4e1c // b.any : list_add_tail(): : * Insert a new entry before the specified head. : * This is useful for implementing queues. : */ : static inline void list_add_tail(struct list_head *new, struct list_head *head) : { : __list_add(new, head->prev, head); 0.00 : ffff8000101d4dfc: ldr x0, [x20, #16] : __list_add(): : new->prev = prev; 0.00 : ffff8000101d4e00: stp x23, x0, [x29, #216] : next->prev = new; 0.00 : ffff8000101d4e04: str x21, [x20, #16] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101d4e08: str x21, [x0] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101d4e0c: b ffff8000101d4f5c 0.00 : ffff8000101d4e10: b ffff8000101d4f5c : __lse_atomic64_or(): 0.00 : ffff8000101d4e14: mov x0, x25 0.00 : ffff8000101d4e18: stset x0, [x19] : get_current(): 0.00 : ffff8000101d4e1c: mrs x0, sp_el0 : __write_once_size(): 0.00 : ffff8000101d4e20: str x24, [x0, #24] : wait_on_page_bit_common(): : set_current_state(state); 0.00 : ffff8000101d4e24: dmb ish : spin_unlock_irq(): : raw_spin_unlock_bh(&lock->rlock); : } : : static __always_inline void spin_unlock_irq(spinlock_t *lock) : { : raw_spin_unlock_irq(&lock->rlock); 0.00 : ffff8000101d4e28: mov x0, x20 0.00 : ffff8000101d4e2c: bl ffff800010cd7c70 <_raw_spin_unlock_irq> : test_bit(): 0.00 : ffff8000101d4e30: ldr x0, [x19] : wait_on_page_bit_common(): : if (likely(bit_is_set)) 0.00 : ffff8000101d4e34: tbz w0, #0, ffff8000101d4e3c : io_schedule(); 0.00 : ffff8000101d4e38: bl ffff800010cd3008 : test_bit(): 0.00 : ffff8000101d4e3c: ldr x0, [x19] : wait_on_page_bit_common(): : if (!test_bit(bit_nr, &page->flags)) 0.00 : ffff8000101d4e40: tbz w0, #0, ffff8000101d5074 : get_current(): 0.00 : ffff8000101d4e44: mrs x0, sp_el0 : test_bit(): 0.00 : ffff8000101d4e48: ldr x1, [x0] : signal_pending_state(): : : static inline int signal_pending_state(long state, struct task_struct *p) : { : if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL))) : return 0; : if (!signal_pending(p)) 0.00 : ffff8000101d4e4c: tbz w1, #0, ffff8000101d4de8 : return 0; : : return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); 0.00 : ffff8000101d4e50: ldr x0, [x0, #1696] 0.00 : ffff8000101d4e54: tbz w0, #8, ffff8000101d4de8 : wait_on_page_bit_common(): : finish_wait(q, wait); 0.00 : ffff8000101d4e58: mov x0, x20 0.00 : ffff8000101d4e5c: add x1, x29, #0xc0 0.00 : ffff8000101d4e60: bl ffff80001012fc00 : ret = -EINTR; 0.00 : ffff8000101d4e64: mov w19, #0xfffffffc // #-4 : if (delayacct) 0.00 : ffff8000101d4e68: ldr w0, [x29, #124] 0.00 : ffff8000101d4e6c: cbnz w0, ffff8000101d5674 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101d4e70: ldr x0, [x22, #8] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000101d4e74: sub x1, x0, #0x1 0.00 : ffff8000101d4e78: tst x0, #0x1 0.00 : ffff8000101d4e7c: csel x22, x1, x22, ne // ne = any : page_ref_dec_and_test(): : return ret; : } : : static inline int page_ref_dec_and_test(struct page *page) : { : int ret = atomic_dec_and_test(&page->_refcount); 0.00 : ffff8000101d4e80: add x1, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d4e84: b ffff8000101d5380 0.00 : ffff8000101d4e88: b ffff8000101d5380 : __lse_atomic_sub_return(): : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000101d4e8c: mov w0, #0x1 // #1 0.00 : ffff8000101d4e90: neg w0, w0 0.00 : ffff8000101d4e94: ldaddal w0, w2, [x1] 0.00 : ffff8000101d4e98: add w0, w0, w2 : put_page(): : if (page_is_devmap_managed(page)) { : put_devmap_managed_page(page); : return; : } : : if (put_page_testzero(page)) 0.00 : ffff8000101d4e9c: cbz w0, ffff8000101d548c : generic_file_buffered_read(): : *ppos = ((loff_t)index << PAGE_SHIFT) + offset; 0.00 : ffff8000101d4ea0: ldr x2, [x29, #128] 0.00 : ffff8000101d4ea4: ldr x1, [x29, #152] : ra->prev_pos |= prev_offset; 0.00 : ffff8000101d4ea8: ldr w0, [x29, #120] : *ppos = ((loff_t)index << PAGE_SHIFT) + offset; 0.00 : ffff8000101d4eac: add x1, x2, x1, lsl #12 : ra->prev_pos |= prev_offset; 0.00 : ffff8000101d4eb0: ldr x2, [x29, #104] 0.00 : ffff8000101d4eb4: orr x0, x0, x2, lsl #12 0.00 : ffff8000101d4eb8: ldr x2, [x29, #136] 0.00 : ffff8000101d4ebc: str x0, [x2, #24] : file_accessed(): 0.00 : ffff8000101d4ec0: ldr x0, [x29, #160] : generic_file_buffered_read(): : *ppos = ((loff_t)index << PAGE_SHIFT) + offset; 0.00 : ffff8000101d4ec4: str x1, [x27, #8] : file_accessed(): 0.00 : ffff8000101d4ec8: ldr w0, [x0, #64] 0.00 : ffff8000101d4ecc: tbz w0, #18, ffff8000101d5448 : generic_file_buffered_read(): : return written ? written : error; 0.00 : ffff8000101d4ed0: ldr x0, [x29, #168] 0.00 : ffff8000101d4ed4: sxtw x19, w19 0.00 : ffff8000101d4ed8: ldp x20, x21, [x29, #24] 0.00 : ffff8000101d4edc: cmp x0, #0x0 0.00 : ffff8000101d4ee0: csel x0, x19, x0, eq // eq = none 0.00 : ffff8000101d4ee4: ldr x26, [x29, #72] 0.00 : ffff8000101d4ee8: ldp x22, x23, [x29, #40] 0.00 : ffff8000101d4eec: str x0, [x29, #168] 0.00 : ffff8000101d4ef0: ldp x24, x25, [x29, #56] 0.00 : ffff8000101d4ef4: b ffff8000101d4f04 0.00 : ffff8000101d4ef8: ldr x20, [x29, #24] 0.00 : ffff8000101d4efc: ldr x26, [x29, #72] : generic_file_read_iter(): : ssize_t retval = 0; 0.00 : ffff8000101d4f00: str xzr, [x29, #168] : } : : retval = generic_file_buffered_read(iocb, iter, retval); : out: : return retval; : } 0.00 : ffff8000101d4f04: adrp x0, ffff800011909000 0.00 : ffff8000101d4f08: add x1, x0, #0x908 0.00 : ffff8000101d4f0c: ldr x2, [x29, #232] 0.00 : ffff8000101d4f10: ldr x1, [x1] 0.00 : ffff8000101d4f14: eor x1, x2, x1 0.95 : ffff8000101d4f18: ldr x0, [x29, #168] 0.00 : ffff8000101d4f1c: cbnz x1, ffff8000101d5658 1.92 : ffff8000101d4f20: ldr x19, [sp, #16] 1.07 : ffff8000101d4f24: ldp x27, x28, [sp, #80] 1.09 : ffff8000101d4f28: ldp x29, x30, [sp], #240 0.00 : ffff8000101d4f2c: ret : if (filemap_range_has_page(mapping, iocb->ki_pos, 0.00 : ffff8000101d4f30: mov x0, x26 0.00 : ffff8000101d4f34: bl ffff8000101d01e0 0.00 : ffff8000101d4f38: tst w0, #0xff 0.00 : ffff8000101d4f3c: b.eq ffff8000101d4bf8 // b.none : return -EAGAIN; 0.00 : ffff8000101d4f40: mov x0, #0xfffffffffffffff5 // #-11 0.00 : ffff8000101d4f44: ldr x20, [x29, #24] 0.00 : ffff8000101d4f48: ldr x26, [x29, #72] 0.00 : ffff8000101d4f4c: str x0, [x29, #168] 0.00 : ffff8000101d4f50: b ffff8000101d4f04 : ssize_t retval = 0; 0.00 : ffff8000101d4f54: str xzr, [x29, #168] 0.00 : ffff8000101d4f58: b ffff8000101d4c68 : __ll_sc_atomic64_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000101d4f5c: b ffff8000101d722c 0.00 : ffff8000101d4f60: b ffff8000101d4e1c : PageUptodate(): : smp_rmb(); 0.00 : ffff8000101d4f64: dmb ishld : generic_file_buffered_read(): : isize = i_size_read(inode); 0.00 : ffff8000101d4f68: ldr x0, [x29, #144] 0.00 : ffff8000101d4f6c: ldr x0, [x0, #80] : end_index = (isize - 1) >> PAGE_SHIFT; 0.00 : ffff8000101d4f70: sub x2, x0, #0x1 : if (unlikely(!isize || index > end_index)) { 0.00 : ffff8000101d4f74: cmp x0, #0x0 0.00 : ffff8000101d4f78: ldr x0, [x29, #152] : end_index = (isize - 1) >> PAGE_SHIFT; 0.00 : ffff8000101d4f7c: asr x1, x2, #12 : if (unlikely(!isize || index > end_index)) { 0.00 : ffff8000101d4f80: ccmp x0, x1, #0x2, ne // ne = any 0.00 : ffff8000101d4f84: b.hi ffff8000101d55f4 // b.pmore : if (index == end_index) { 0.00 : ffff8000101d4f88: ldr x0, [x29, #152] : nr = PAGE_SIZE; 0.00 : ffff8000101d4f8c: mov x19, #0x1000 // #4096 : if (index == end_index) { 0.00 : ffff8000101d4f90: cmp x0, x1 0.00 : ffff8000101d4f94: b.eq ffff8000101d5558 // b.none : __read_once_size(): 0.00 : ffff8000101d4f98: ldr w0, [x26, #28] : generic_file_buffered_read(): : nr = nr - offset; 0.00 : ffff8000101d4f9c: ldr x1, [x29, #128] : if (mapping_writably_mapped(mapping)) 0.00 : ffff8000101d4fa0: cmp w0, #0x0 : nr = nr - offset; 0.00 : ffff8000101d4fa4: sub x19, x19, x1 : if (mapping_writably_mapped(mapping)) 0.00 : ffff8000101d4fa8: b.gt ffff8000101d554c : if (prev_index != index || offset != prev_offset) 0.00 : ffff8000101d4fac: ldr x1, [x29, #104] 0.00 : ffff8000101d4fb0: ldr x0, [x29, #152] 0.00 : ffff8000101d4fb4: cmp x0, x1 0.00 : ffff8000101d4fb8: b.ne ffff8000101d4fcc // b.any 0.00 : ffff8000101d4fbc: ldr w0, [x29, #120] 0.00 : ffff8000101d4fc0: ldr x1, [x29, #128] 0.00 : ffff8000101d4fc4: cmp x1, x0 0.00 : ffff8000101d4fc8: b.eq ffff8000101d4fd4 // b.none : mark_page_accessed(page); 0.00 : ffff8000101d4fcc: mov x0, x22 0.00 : ffff8000101d4fd0: bl ffff8000101e1b30 : ret = copy_page_to_iter(page, offset, nr, iter); 0.00 : ffff8000101d4fd4: ldr x20, [x29, #128] 0.00 : ffff8000101d4fd8: mov x0, x22 0.00 : ffff8000101d4fdc: mov x3, x28 0.00 : ffff8000101d4fe0: mov x2, x19 0.00 : ffff8000101d4fe4: mov x1, x20 0.00 : ffff8000101d4fe8: bl ffff80001048b3b8 : __read_once_size(): 0.00 : ffff8000101d4fec: ldr x1, [x22, #8] : generic_file_buffered_read(): 0.00 : ffff8000101d4ff0: mov x23, x0 : index += offset >> PAGE_SHIFT; 0.00 : ffff8000101d4ff4: ldr x3, [x29, #152] : offset += ret; 0.00 : ffff8000101d4ff8: add x0, x20, x23 : offset &= ~PAGE_MASK; 0.00 : ffff8000101d4ffc: and x2, x0, #0xfff : prev_offset = offset; 0.00 : ffff8000101d5000: str w2, [x29, #120] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000101d5004: tst x1, #0x1 0.00 : ffff8000101d5008: sub x1, x1, #0x1 0.00 : ffff8000101d500c: csel x22, x1, x22, ne // ne = any : generic_file_buffered_read(): : index += offset >> PAGE_SHIFT; 0.00 : ffff8000101d5010: add x20, x3, x0, lsr #12 : offset &= ~PAGE_MASK; 0.00 : ffff8000101d5014: str x2, [x29, #128] : page_ref_dec_and_test(): 0.00 : ffff8000101d5018: add x0, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d501c: b ffff8000101d55a4 0.00 : ffff8000101d5020: b ffff8000101d55a4 : __lse_atomic_sub_return(): 0.00 : ffff8000101d5024: mov w1, #0x1 // #1 0.00 : ffff8000101d5028: neg w1, w1 0.00 : ffff8000101d502c: ldaddal w1, w2, [x0] 0.00 : ffff8000101d5030: add w1, w1, w2 0.00 : ffff8000101d5034: mov w0, w1 : put_page(): 0.00 : ffff8000101d5038: cbnz w0, ffff8000101d5044 : __put_page(page); 0.00 : ffff8000101d503c: mov x0, x22 0.00 : ffff8000101d5040: bl ffff8000101dfec0 <__put_page> : generic_file_buffered_read(): : written += ret; 0.00 : ffff8000101d5044: ldr x1, [x29, #168] : if (!iov_iter_count(iter)) 0.00 : ffff8000101d5048: ldr x0, [x28, #16] : written += ret; 0.00 : ffff8000101d504c: add x1, x1, x23 0.00 : ffff8000101d5050: str x1, [x29, #168] : if (!iov_iter_count(iter)) 0.00 : ffff8000101d5054: cbz x0, ffff8000101d5644 : if (ret < nr) { 0.00 : ffff8000101d5058: ldr x0, [x29, #152] 0.00 : ffff8000101d505c: cmp x19, x23 0.00 : ffff8000101d5060: str x0, [x29, #104] 0.00 : ffff8000101d5064: b.ls ffff8000101d55b4 // b.plast : error = -EFAULT; 0.00 : ffff8000101d5068: mov w19, #0xfffffff2 // #-14 : index += offset >> PAGE_SHIFT; 0.00 : ffff8000101d506c: str x20, [x29, #152] 0.00 : ffff8000101d5070: b ffff8000101d4ea0 : wait_on_page_bit_common(): : finish_wait(q, wait); 0.00 : ffff8000101d5074: mov x0, x20 0.00 : ffff8000101d5078: add x1, x29, #0xc0 0.00 : ffff8000101d507c: bl ffff80001012fc00 : if (delayacct) 0.00 : ffff8000101d5080: ldr w0, [x29, #124] 0.00 : ffff8000101d5084: cbz w0, ffff8000101d508c 0.00 : ffff8000101d5088: bl ffff8000101d0650 : __read_once_size(): 0.00 : ffff8000101d508c: ldr x0, [x22, #8] : compound_head(): 0.00 : ffff8000101d5090: tst x0, #0x1 0.00 : ffff8000101d5094: sub x0, x0, #0x1 0.00 : ffff8000101d5098: csel x0, x0, x22, ne // ne = any : test_bit(): 0.00 : ffff8000101d509c: ldr x0, [x0] : PageUptodate(): : if (ret) 0.00 : ffff8000101d50a0: tbnz w0, #2, ffff8000101d4f64 : generic_file_buffered_read(): : if (inode->i_blkbits == PAGE_SHIFT || 0.00 : ffff8000101d50a4: ldr x0, [x29, #144] 0.00 : ffff8000101d50a8: ldrb w0, [x0, #142] 0.00 : ffff8000101d50ac: cmp w0, #0xc 0.00 : ffff8000101d50b0: b.eq ffff8000101d50e8 // b.none : !mapping->a_ops->is_partially_uptodate) 0.00 : ffff8000101d50b4: ldr x0, [x26, #112] : if (inode->i_blkbits == PAGE_SHIFT || 0.00 : ffff8000101d50b8: ldr x0, [x0, #128] 0.00 : ffff8000101d50bc: cbz x0, ffff8000101d50e8 : iov_iter_type(): : return i->type & ~(READ | WRITE); 0.00 : ffff8000101d50c0: ldr w0, [x28] 0.00 : ffff8000101d50c4: and w0, w0, #0xfffffffe : generic_file_buffered_read(): : if (unlikely(iov_iter_is_pipe(iter))) 0.00 : ffff8000101d50c8: cmp w0, #0x20 0.00 : ffff8000101d50cc: b.eq ffff8000101d50e8 // b.none : __read_once_size(): 0.00 : ffff8000101d50d0: ldr x0, [x22, #8] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000101d50d4: tst x0, #0x1 0.00 : ffff8000101d50d8: sub x0, x0, #0x1 0.00 : ffff8000101d50dc: csel x0, x0, x22, ne // ne = any : __read_once_size(): 0.00 : ffff8000101d50e0: ldr x1, [x0] : test_and_set_bit_lock(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (READ_ONCE(*p) & mask) 0.00 : ffff8000101d50e4: tbz w1, #0, ffff8000101d53b8 : __read_once_size(): 0.00 : ffff8000101d50e8: ldr x0, [x22, #8] : compound_head(): 0.00 : ffff8000101d50ec: tst x0, #0x1 0.00 : ffff8000101d50f0: sub x0, x0, #0x1 0.00 : ffff8000101d50f4: csel x0, x0, x22, ne // ne = any : __read_once_size(): 0.00 : ffff8000101d50f8: ldr x1, [x0] : test_and_set_bit_lock(): 0.00 : ffff8000101d50fc: tbz w1, #0, ffff8000101d538c : lock_page_killable(): : return __lock_page_killable(page); 0.00 : ffff8000101d5100: mov x0, x22 0.00 : ffff8000101d5104: bl ffff8000101d1678 <__lock_page_killable> 0.00 : ffff8000101d5108: mov w19, w0 : generic_file_buffered_read(): : if (unlikely(error)) 0.00 : ffff8000101d510c: cbnz w0, ffff8000101d4e70 : if (!page->mapping) { 0.00 : ffff8000101d5110: ldr x0, [x22, #24] 0.00 : ffff8000101d5114: cbz x0, ffff8000101d5498 : __read_once_size(): 0.00 : ffff8000101d5118: ldr x0, [x22, #8] : compound_head(): 0.00 : ffff8000101d511c: tst x0, #0x1 0.00 : ffff8000101d5120: sub x0, x0, #0x1 0.00 : ffff8000101d5124: csel x0, x0, x22, ne // ne = any : test_bit(): 0.00 : ffff8000101d5128: ldr x0, [x0] : PageUptodate(): : if (ret) 0.00 : ffff8000101d512c: tbz w0, #2, ffff8000101d5140 : smp_rmb(); 0.00 : ffff8000101d5130: dmb ishld : generic_file_buffered_read(): : unlock_page(page); 0.00 : ffff8000101d5134: mov x0, x22 0.00 : ffff8000101d5138: bl ffff8000101d1588 0.00 : ffff8000101d513c: b ffff8000101d4f68 : ClearPageError(): : PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND) 0.00 : ffff8000101d5140: mov x20, x22 : arch_static_branch_jump(): 0.00 : ffff8000101d5144: b ffff8000101d5210 0.00 : ffff8000101d5148: b ffff8000101d5210 : __lse_atomic64_andnot(): : ATOMIC64_OP(andnot, stclr) 0.00 : ffff8000101d514c: mov x0, #0x100 // #256 0.00 : ffff8000101d5150: stclr x0, [x22] : generic_file_buffered_read(): : error = mapping->a_ops->readpage(filp, page); 0.00 : ffff8000101d5154: ldr x0, [x26, #112] 0.00 : ffff8000101d5158: mov x1, x22 0.00 : ffff8000101d515c: ldr x2, [x0, #8] 0.00 : ffff8000101d5160: ldr x0, [x29, #160] 0.00 : ffff8000101d5164: blr x2 0.00 : ffff8000101d5168: mov w19, w0 : if (unlikely(error)) { 0.00 : ffff8000101d516c: cbnz w0, ffff8000101d54ec : __read_once_size(): 0.00 : ffff8000101d5170: ldr x0, [x22, #8] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000101d5174: tst x0, #0x1 0.00 : ffff8000101d5178: sub x0, x0, #0x1 0.00 : ffff8000101d517c: csel x0, x0, x22, ne // ne = any : test_bit(): 0.00 : ffff8000101d5180: ldr x0, [x0] : PageUptodate(): : if (ret) 0.00 : ffff8000101d5184: tbnz w0, #2, ffff8000101d4f64 : __read_once_size(): 0.00 : ffff8000101d5188: ldr x0, [x22, #8] : compound_head(): : if (unlikely(head & 1)) 0.00 : ffff8000101d518c: tbnz w0, #0, ffff8000101d5540 0.00 : ffff8000101d5190: mov x0, x22 : __read_once_size(): 0.00 : ffff8000101d5194: ldr x0, [x0] : test_and_set_bit_lock(): 0.00 : ffff8000101d5198: tbz w0, #0, ffff8000101d5350 : lock_page_killable(): 0.00 : ffff8000101d519c: mov x0, x22 0.00 : ffff8000101d51a0: bl ffff8000101d1678 <__lock_page_killable> 0.00 : ffff8000101d51a4: mov w19, w0 : generic_file_buffered_read(): : if (unlikely(error)) 0.00 : ffff8000101d51a8: cbnz w0, ffff8000101d4e70 : __read_once_size(): 0.00 : ffff8000101d51ac: ldr x0, [x22, #8] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000101d51b0: tst x0, #0x1 0.00 : ffff8000101d51b4: sub x0, x0, #0x1 0.00 : ffff8000101d51b8: csel x0, x0, x22, ne // ne = any : test_bit(): 0.00 : ffff8000101d51bc: ldr x0, [x0] : PageUptodate(): : if (ret) 0.00 : ffff8000101d51c0: tbnz w0, #2, ffff8000101d5130 : generic_file_buffered_read(): : if (page->mapping == NULL) { 0.00 : ffff8000101d51c4: ldr x0, [x22, #24] 0.00 : ffff8000101d51c8: cbnz x0, ffff8000101d55c8 : unlock_page(page); 0.00 : ffff8000101d51cc: mov x0, x22 0.00 : ffff8000101d51d0: bl ffff8000101d1588 : __read_once_size(): 0.00 : ffff8000101d51d4: ldr x0, [x22, #8] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000101d51d8: tst x0, #0x1 0.00 : ffff8000101d51dc: sub x0, x0, #0x1 0.00 : ffff8000101d51e0: csel x22, x0, x22, ne // ne = any : page_ref_dec_and_test(): 0.00 : ffff8000101d51e4: add x0, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d51e8: b ffff8000101d521c 0.00 : ffff8000101d51ec: b ffff8000101d521c : __lse_atomic_sub_return(): : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000101d51f0: mov w1, #0x1 // #1 0.00 : ffff8000101d51f4: neg w1, w1 0.00 : ffff8000101d51f8: ldaddal w1, w2, [x0] 0.00 : ffff8000101d51fc: add w1, w1, w2 : put_page(): : if (put_page_testzero(page)) 0.00 : ffff8000101d5200: cbnz w1, ffff8000101d4ce8 : __put_page(page); 0.00 : ffff8000101d5204: mov x0, x22 0.00 : ffff8000101d5208: bl ffff8000101dfec0 <__put_page> 0.00 : ffff8000101d520c: b ffff8000101d4ce8 : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff8000101d5210: mov x2, #0x100 // #256 0.00 : ffff8000101d5214: b ffff8000101d7244 0.00 : ffff8000101d5218: b ffff8000101d5154 : __ll_sc_atomic_sub_return(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000101d521c: mov w2, #0x1 // #1 0.00 : ffff8000101d5220: b ffff8000101d725c 0.00 : ffff8000101d5224: b ffff8000101d5200 : generic_file_buffered_read(): : page_cache_async_readahead(mapping, 0.00 : ffff8000101d5228: ldp x0, x2, [x29, #152] 0.00 : ffff8000101d522c: mov x3, x22 0.00 : ffff8000101d5230: ldr x1, [x29, #112] 0.00 : ffff8000101d5234: mov x4, x0 0.00 : ffff8000101d5238: sub x5, x1, x0 0.00 : ffff8000101d523c: ldr x1, [x29, #136] 0.00 : ffff8000101d5240: mov x0, x26 0.00 : ffff8000101d5244: bl ffff8000101df650 0.00 : ffff8000101d5248: b ffff8000101d4d18 : if (iocb->ki_flags & IOCB_NOWAIT) 0.00 : ffff8000101d524c: ldr w0, [x27, #32] 0.00 : ffff8000101d5250: tbnz w0, #7, ffff8000101d563c : page_cache_sync_readahead(mapping, 0.00 : ffff8000101d5254: ldp x19, x2, [x29, #152] 0.00 : ffff8000101d5258: mov x0, x26 0.00 : ffff8000101d525c: ldr x1, [x29, #112] 0.00 : ffff8000101d5260: mov x3, x19 0.00 : ffff8000101d5264: sub x4, x1, x19 0.00 : ffff8000101d5268: ldr x1, [x29, #136] 0.00 : ffff8000101d526c: bl ffff8000101df840 : find_get_page(): : return pagecache_get_page(mapping, offset, 0, 0); 0.00 : ffff8000101d5270: mov w3, #0x0 // #0 0.00 : ffff8000101d5274: mov w2, #0x0 // #0 0.00 : ffff8000101d5278: mov x1, x19 0.00 : ffff8000101d527c: mov x0, x26 0.00 : ffff8000101d5280: bl ffff8000101d3118 0.00 : ffff8000101d5284: mov x22, x0 : generic_file_buffered_read(): : if (unlikely(page == NULL)) 0.00 : ffff8000101d5288: cbnz x0, ffff8000101d4d10 : page_cache_alloc(): : return __page_cache_alloc(mapping_gfp_mask(x)); 0.00 : ffff8000101d528c: ldr w0, [x26, #24] 0.00 : ffff8000101d5290: bl ffff8000101d05a0 <__page_cache_alloc> 0.00 : ffff8000101d5294: mov x22, x0 : generic_file_buffered_read(): : if (!page) { 0.00 : ffff8000101d5298: cbz x0, ffff8000101d566c : error = add_to_page_cache_lru(page, mapping, index, 0.00 : ffff8000101d529c: ldr x2, [x29, #152] : mapping_gfp_constraint(): : return mapping_gfp_mask(mapping) & gfp_mask; 0.00 : ffff8000101d52a0: mov w1, #0xcc0 // #3264 0.00 : ffff8000101d52a4: ldr w3, [x26, #24] : generic_file_buffered_read(): 0.00 : ffff8000101d52a8: and w3, w3, w1 0.00 : ffff8000101d52ac: mov x1, x26 0.00 : ffff8000101d52b0: bl ffff8000101d28c8 0.00 : ffff8000101d52b4: mov w19, w0 : if (error) { 0.00 : ffff8000101d52b8: cbz w0, ffff8000101d5140 : __read_once_size(): 0.00 : ffff8000101d52bc: ldr x0, [x22, #8] : compound_head(): 0.00 : ffff8000101d52c0: tst x0, #0x1 0.00 : ffff8000101d52c4: sub x0, x0, #0x1 0.00 : ffff8000101d52c8: csel x22, x0, x22, ne // ne = any : page_ref_dec_and_test(): 0.00 : ffff8000101d52cc: add x1, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d52d0: b ffff8000101d55e8 0.00 : ffff8000101d52d4: b ffff8000101d55e8 : __lse_atomic_sub_return(): 0.00 : ffff8000101d52d8: mov w0, #0x1 // #1 0.00 : ffff8000101d52dc: neg w0, w0 0.00 : ffff8000101d52e0: ldaddal w0, w2, [x1] 0.00 : ffff8000101d52e4: add w0, w0, w2 : put_page(): : if (put_page_testzero(page)) 0.00 : ffff8000101d52e8: cbnz w0, ffff8000101d52f4 : __put_page(page); 0.00 : ffff8000101d52ec: mov x0, x22 0.00 : ffff8000101d52f0: bl ffff8000101dfec0 <__put_page> : generic_file_buffered_read(): : if (error == -EEXIST) { 0.00 : ffff8000101d52f4: cmn w19, #0x11 0.00 : ffff8000101d52f8: b.eq ffff8000101d4ce8 // b.none 0.00 : ffff8000101d52fc: b ffff8000101d4ea0 : __read_once_size(): 0.00 : ffff8000101d5300: ldr x0, [x19, #8] : wait_on_page_bit_common(): : !PageUptodate(page) && PageWorkingset(page)) { 0.00 : ffff8000101d5304: str wzr, [x29, #124] : compound_head(): : return page; 0.00 : ffff8000101d5308: tst x0, #0x1 0.00 : ffff8000101d530c: sub x0, x0, #0x1 0.00 : ffff8000101d5310: csel x0, x0, x19, ne // ne = any : test_bit(): 0.00 : ffff8000101d5314: ldr x0, [x0] : wait_on_page_bit_common(): 0.00 : ffff8000101d5318: tbz w0, #6, ffff8000101d4dac : __read_once_size(): 0.00 : ffff8000101d531c: ldr x1, [x19, #8] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000101d5320: tst x1, #0x1 0.00 : ffff8000101d5324: sub x1, x1, #0x1 0.00 : ffff8000101d5328: csel x1, x1, x19, ne // ne = any : test_bit(): 0.00 : ffff8000101d532c: ldr x0, [x1] : wait_on_page_bit_common(): : if (!PageSwapBacked(page)) { 0.00 : ffff8000101d5330: tbnz w0, #19, ffff8000101d4dac : get_current(): 0.00 : ffff8000101d5334: mrs x0, sp_el0 : delayacct_thrashing_start(): : __delayacct_freepages_end(); : } : : static inline void delayacct_thrashing_start(void) : { : if (current->delays) 0.00 : ffff8000101d5338: ldr x0, [x0, #2368] 0.00 : ffff8000101d533c: mov w1, #0x1 // #1 0.00 : ffff8000101d5340: str w1, [x29, #124] 0.00 : ffff8000101d5344: cbz x0, ffff8000101d4dac : __delayacct_thrashing_start(); 0.00 : ffff8000101d5348: bl ffff8000101b2eb0 <__delayacct_thrashing_start> 0.00 : ffff8000101d534c: b ffff8000101d4dac : arch_static_branch_jump(): 0.00 : ffff8000101d5350: b ffff8000101d5368 0.00 : ffff8000101d5354: b ffff8000101d5368 : __lse_atomic64_fetch_or_acquire(): : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff8000101d5358: mov x0, #0x1 // #1 0.00 : ffff8000101d535c: ldseta x0, x0, [x20] : lock_page_killable(): : if (!trylock_page(page)) 0.00 : ffff8000101d5360: tbz w0, #0, ffff8000101d51ac 0.00 : ffff8000101d5364: b ffff8000101d519c : __ll_sc_atomic64_fetch_or_acquire(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000101d5368: b ffff8000101d7278 : lock_page_killable(): 0.00 : ffff8000101d536c: tbz w0, #0, ffff8000101d51ac 0.00 : ffff8000101d5370: b ffff8000101d519c 0.00 : ffff8000101d5374: ldr x20, [x29, #24] 0.00 : ffff8000101d5378: ldr x26, [x29, #72] 0.00 : ffff8000101d537c: b ffff8000101d4f04 : __ll_sc_atomic_sub_return(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000101d5380: mov w2, #0x1 // #1 0.00 : ffff8000101d5384: b ffff8000101d7290 0.00 : ffff8000101d5388: b ffff8000101d4e9c : arch_static_branch_jump(): 0.00 : ffff8000101d538c: b ffff8000101d53a8 0.00 : ffff8000101d5390: b ffff8000101d53a8 : __lse_atomic64_fetch_or_acquire(): 0.00 : ffff8000101d5394: mov x1, #0x1 // #1 0.00 : ffff8000101d5398: ldseta x1, x1, [x0] 0.00 : ffff8000101d539c: mov x0, x1 : lock_page_killable(): 0.00 : ffff8000101d53a0: tbz w0, #0, ffff8000101d5110 0.00 : ffff8000101d53a4: b ffff8000101d5100 : __ll_sc_atomic64_fetch_or_acquire(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000101d53a8: b ffff8000101d72ac 0.00 : ffff8000101d53ac: mov x0, x1 : lock_page_killable(): 0.00 : ffff8000101d53b0: tbz w0, #0, ffff8000101d5110 0.00 : ffff8000101d53b4: b ffff8000101d5100 : arch_static_branch_jump(): 0.00 : ffff8000101d53b8: b ffff8000101d5400 0.00 : ffff8000101d53bc: b ffff8000101d5400 : __lse_atomic64_fetch_or_acquire(): 0.00 : ffff8000101d53c0: mov x1, #0x1 // #1 0.00 : ffff8000101d53c4: ldseta x1, x1, [x0] 0.00 : ffff8000101d53c8: mov x0, x1 : generic_file_buffered_read(): : if (!trylock_page(page)) 0.00 : ffff8000101d53cc: tbnz w0, #0, ffff8000101d50e8 : if (!page->mapping) 0.00 : ffff8000101d53d0: ldr x0, [x22, #24] 0.00 : ffff8000101d53d4: cbz x0, ffff8000101d5498 : if (!mapping->a_ops->is_partially_uptodate(page, 0.00 : ffff8000101d53d8: ldr x0, [x26, #112] 0.00 : ffff8000101d53dc: ldr x2, [x28, #16] 0.00 : ffff8000101d53e0: ldr x1, [x29, #128] 0.00 : ffff8000101d53e4: ldr x3, [x0, #128] 0.00 : ffff8000101d53e8: mov x0, x22 0.00 : ffff8000101d53ec: blr x3 0.00 : ffff8000101d53f0: cbz w0, ffff8000101d5110 : unlock_page(page); 0.00 : ffff8000101d53f4: mov x0, x22 0.00 : ffff8000101d53f8: bl ffff8000101d1588 0.00 : ffff8000101d53fc: b ffff8000101d4f68 : __ll_sc_atomic64_fetch_or_acquire(): 0.00 : ffff8000101d5400: b ffff8000101d72c4 0.00 : ffff8000101d5404: mov x0, x1 0.00 : ffff8000101d5408: b ffff8000101d53cc : generic_file_read_iter(): : iov_iter_revert(iter, count - iov_iter_count(iter)); 18.14 : ffff8000101d540c: ldr x1, [x28, #16] 0.00 : ffff8000101d5410: mov x0, x28 0.00 : ffff8000101d5414: sub x1, x19, x1 0.00 : ffff8000101d5418: bl ffff800010488720 11.82 : ffff8000101d541c: ldr x20, [x29, #24] 6.41 : ffff8000101d5420: ldr x26, [x29, #72] 0.00 : ffff8000101d5424: b ffff8000101d4f04 : file_accessed(): : touch_atime(&file->f_path); 0.00 : ffff8000101d5428: ldr x0, [x29, #160] 0.00 : ffff8000101d542c: add x0, x0, #0x10 0.00 : ffff8000101d5430: bl ffff8000102a0368 0.00 : ffff8000101d5434: b ffff8000101d4c04 : fatal_signal_pending(): : return signal_pending(p) && __fatal_signal_pending(p); 0.00 : ffff8000101d5438: ldr x0, [x0, #1696] 0.00 : ffff8000101d543c: tbz w0, #8, ffff8000101d4cf4 : generic_file_buffered_read(): : error = -EINTR; 0.00 : ffff8000101d5440: mov w19, #0xfffffffc // #-4 0.00 : ffff8000101d5444: b ffff8000101d4ea0 : file_accessed(): 0.00 : ffff8000101d5448: ldr x0, [x29, #160] 0.00 : ffff8000101d544c: add x0, x0, #0x10 0.00 : ffff8000101d5450: bl ffff8000102a0368 0.00 : ffff8000101d5454: b ffff8000101d4ed0 : __read_once_size(): 0.00 : ffff8000101d5458: ldr x0, [x22, #8] : compound_head(): 0.00 : ffff8000101d545c: sub x1, x0, #0x1 0.00 : ffff8000101d5460: tst x0, #0x1 0.00 : ffff8000101d5464: csel x22, x1, x22, ne // ne = any : page_ref_dec_and_test(): 0.00 : ffff8000101d5468: add x1, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d546c: b ffff8000101d55bc 0.00 : ffff8000101d5470: b ffff8000101d55bc : __lse_atomic_sub_return(): : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000101d5474: mov w0, #0x1 // #1 0.00 : ffff8000101d5478: neg w0, w0 0.00 : ffff8000101d547c: ldaddal w0, w2, [x1] 0.00 : ffff8000101d5480: add w0, w0, w2 : generic_file_buffered_read(): : error = -EAGAIN; 0.00 : ffff8000101d5484: mov w19, #0xfffffff5 // #-11 : put_page(): : if (put_page_testzero(page)) 0.00 : ffff8000101d5488: cbnz w0, ffff8000101d4ea0 : __put_page(page); 0.00 : ffff8000101d548c: mov x0, x22 0.00 : ffff8000101d5490: bl ffff8000101dfec0 <__put_page> 0.00 : ffff8000101d5494: b ffff8000101d4ea0 : generic_file_buffered_read(): : unlock_page(page); 0.00 : ffff8000101d5498: mov x0, x22 0.00 : ffff8000101d549c: bl ffff8000101d1588 : __read_once_size(): 0.00 : ffff8000101d54a0: ldr x0, [x22, #8] : compound_head(): 0.00 : ffff8000101d54a4: sub x1, x0, #0x1 0.00 : ffff8000101d54a8: tst x0, #0x1 0.00 : ffff8000101d54ac: csel x22, x1, x22, ne // ne = any : page_ref_dec_and_test(): 0.00 : ffff8000101d54b0: add x0, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d54b4: b ffff8000101d54d8 0.00 : ffff8000101d54b8: b ffff8000101d54d8 : __lse_atomic_sub_return(): 0.00 : ffff8000101d54bc: mov w1, #0x1 // #1 0.00 : ffff8000101d54c0: neg w1, w1 0.00 : ffff8000101d54c4: ldaddal w1, w2, [x0] 0.00 : ffff8000101d54c8: add w1, w1, w2 0.00 : ffff8000101d54cc: mov w0, w1 : put_page(): : if (put_page_testzero(page)) 0.00 : ffff8000101d54d0: cbnz w0, ffff8000101d4ce8 0.00 : ffff8000101d54d4: b ffff8000101d5204 : __ll_sc_atomic_sub_return(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000101d54d8: mov w2, #0x1 // #1 0.00 : ffff8000101d54dc: b ffff8000101d72dc 0.00 : ffff8000101d54e0: mov w0, w1 : put_page(): 0.00 : ffff8000101d54e4: cbnz w0, ffff8000101d4ce8 0.00 : ffff8000101d54e8: b ffff8000101d5204 : generic_file_buffered_read(): : if (error == AOP_TRUNCATED_PAGE) { 0.00 : ffff8000101d54ec: mov w0, #0x1 // #1 0.00 : ffff8000101d54f0: movk w0, #0x8, lsl #16 0.00 : ffff8000101d54f4: cmp w19, w0 0.00 : ffff8000101d54f8: b.ne ffff8000101d4e70 // b.any : __read_once_size(): 0.00 : ffff8000101d54fc: ldr x0, [x22, #8] : compound_head(): 0.00 : ffff8000101d5500: tst x0, #0x1 0.00 : ffff8000101d5504: sub x0, x0, #0x1 0.00 : ffff8000101d5508: csel x22, x0, x22, ne // ne = any : page_ref_dec_and_test(): 0.00 : ffff8000101d550c: add x1, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d5510: b ffff8000101d5530 0.00 : ffff8000101d5514: b ffff8000101d5530 : __lse_atomic_sub_return(): 0.00 : ffff8000101d5518: mov w0, #0x1 // #1 0.00 : ffff8000101d551c: neg w0, w0 0.00 : ffff8000101d5520: ldaddal w0, w2, [x1] 0.00 : ffff8000101d5524: add w0, w0, w2 : put_page(): 0.00 : ffff8000101d5528: cbnz w0, ffff8000101d4ce8 0.00 : ffff8000101d552c: b ffff8000101d5204 : __ll_sc_atomic_sub_return(): 0.00 : ffff8000101d5530: mov w2, #0x1 // #1 0.00 : ffff8000101d5534: b ffff8000101d72f8 : put_page(): 0.00 : ffff8000101d5538: cbnz w0, ffff8000101d4ce8 0.00 : ffff8000101d553c: b ffff8000101d5204 : compound_head(): 0.00 : ffff8000101d5540: sub x20, x0, #0x1 0.00 : ffff8000101d5544: mov x0, x20 0.00 : ffff8000101d5548: b ffff8000101d5194 : generic_file_buffered_read(): : flush_dcache_page(page); 0.00 : ffff8000101d554c: mov x0, x22 0.00 : ffff8000101d5550: bl ffff8000100a3110 0.00 : ffff8000101d5554: b ffff8000101d4fac : nr = ((isize - 1) & ~PAGE_MASK) + 1; 0.00 : ffff8000101d5558: and x2, x2, #0xfff : if (nr <= offset) { 0.00 : ffff8000101d555c: ldr x0, [x29, #128] : nr = ((isize - 1) & ~PAGE_MASK) + 1; 0.00 : ffff8000101d5560: add x19, x2, #0x1 : if (nr <= offset) { 0.00 : ffff8000101d5564: cmp x0, x19 0.00 : ffff8000101d5568: b.cc ffff8000101d4f98 // b.lo, b.ul, b.last : __read_once_size(): 0.00 : ffff8000101d556c: ldr x0, [x22, #8] : compound_head(): : if (unlikely(head & 1)) 0.00 : ffff8000101d5570: tbz w0, #0, ffff8000101d5578 : return (struct page *) (head - 1); 0.00 : ffff8000101d5574: sub x22, x0, #0x1 : page_ref_dec_and_test(): 0.00 : ffff8000101d5578: add x0, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d557c: b ffff8000101d5690 0.00 : ffff8000101d5580: b ffff8000101d5690 : __lse_atomic_sub_return(): 0.00 : ffff8000101d5584: mov w19, #0x1 // #1 0.00 : ffff8000101d5588: neg w19, w19 0.00 : ffff8000101d558c: ldaddal w19, w2, [x0] 0.00 : ffff8000101d5590: add w19, w19, w2 : put_page(): 0.00 : ffff8000101d5594: cbz w19, ffff8000101d5680 : generic_file_buffered_read(): : end_index = (isize - 1) >> PAGE_SHIFT; 0.00 : ffff8000101d5598: str x1, [x29, #152] : put_page(): 0.00 : ffff8000101d559c: mov w19, #0x0 // #0 0.00 : ffff8000101d55a0: b ffff8000101d4ea0 : __ll_sc_atomic_sub_return(): 0.00 : ffff8000101d55a4: mov w2, #0x1 // #1 0.00 : ffff8000101d55a8: b ffff8000101d7314 0.00 : ffff8000101d55ac: mov w0, w1 0.00 : ffff8000101d55b0: b ffff8000101d5038 : generic_file_buffered_read(): : index += offset >> PAGE_SHIFT; 0.00 : ffff8000101d55b4: str x20, [x29, #152] 0.00 : ffff8000101d55b8: b ffff8000101d4ce8 : __ll_sc_atomic_sub_return(): 0.00 : ffff8000101d55bc: mov w2, #0x1 // #1 0.00 : ffff8000101d55c0: b ffff8000101d7330 0.00 : ffff8000101d55c4: b ffff8000101d5484 : generic_file_buffered_read(): : unlock_page(page); 0.00 : ffff8000101d55c8: mov x0, x22 0.00 : ffff8000101d55cc: bl ffff8000101d1588 : shrink_readahead_size_eio(): : ra->ra_pages /= 4; 0.00 : ffff8000101d55d0: ldr x1, [x29, #160] : generic_file_buffered_read(): : error = -EIO; 0.00 : ffff8000101d55d4: mov w19, #0xfffffffb // #-5 : shrink_readahead_size_eio(): : ra->ra_pages /= 4; 0.00 : ffff8000101d55d8: ldr w0, [x1, #168] 0.00 : ffff8000101d55dc: lsr w0, w0, #2 0.00 : ffff8000101d55e0: str w0, [x1, #168] 0.00 : ffff8000101d55e4: b ffff8000101d4e70 : __ll_sc_atomic_sub_return(): 0.00 : ffff8000101d55e8: mov w2, #0x1 // #1 0.00 : ffff8000101d55ec: b ffff8000101d734c 0.00 : ffff8000101d55f0: b ffff8000101d52e8 : __read_once_size(): 0.00 : ffff8000101d55f4: ldr x0, [x22, #8] : compound_head(): 0.00 : ffff8000101d55f8: sub x1, x0, #0x1 0.00 : ffff8000101d55fc: tst x0, #0x1 0.00 : ffff8000101d5600: csel x22, x1, x22, ne // ne = any : page_ref_dec_and_test(): 0.00 : ffff8000101d5604: add x0, x22, #0x34 : arch_static_branch_jump(): 0.00 : ffff8000101d5608: b ffff8000101d562c 0.00 : ffff8000101d560c: b ffff8000101d562c : __lse_atomic_sub_return(): 0.00 : ffff8000101d5610: mov w19, #0x1 // #1 0.00 : ffff8000101d5614: neg w19, w19 0.00 : ffff8000101d5618: ldaddal w19, w1, [x0] 0.00 : ffff8000101d561c: add w19, w19, w1 : put_page(): 0.00 : ffff8000101d5620: cbz w19, ffff8000101d548c 0.00 : ffff8000101d5624: mov w19, #0x0 // #0 0.00 : ffff8000101d5628: b ffff8000101d4ea0 : __ll_sc_atomic_sub_return(): 0.00 : ffff8000101d562c: mov w1, #0x1 // #1 0.00 : ffff8000101d5630: b ffff8000101d7368 : put_page(): 0.00 : ffff8000101d5634: cbnz w19, ffff8000101d5624 0.00 : ffff8000101d5638: b ffff8000101d548c : generic_file_buffered_read(): : error = -EAGAIN; 0.00 : ffff8000101d563c: mov w19, #0xfffffff5 // #-11 0.00 : ffff8000101d5640: b ffff8000101d4ea0 : if (!iov_iter_count(iter)) 0.00 : ffff8000101d5644: ldr x0, [x29, #152] 0.00 : ffff8000101d5648: mov w19, #0x0 // #0 0.00 : ffff8000101d564c: str x0, [x29, #104] : index += offset >> PAGE_SHIFT; 0.00 : ffff8000101d5650: str x20, [x29, #152] 0.00 : ffff8000101d5654: b ffff8000101d4ea0 0.00 : ffff8000101d5658: stp x20, x21, [x29, #24] 0.00 : ffff8000101d565c: stp x22, x23, [x29, #40] 0.00 : ffff8000101d5660: stp x24, x25, [x29, #56] 0.00 : ffff8000101d5664: str x26, [x29, #72] : generic_file_read_iter(): : } 0.00 : ffff8000101d5668: bl ffff8000100e64d0 <__stack_chk_fail> : generic_file_buffered_read(): : error = -ENOMEM; 0.00 : ffff8000101d566c: mov w19, #0xfffffff4 // #-12 0.00 : ffff8000101d5670: b ffff8000101d4ea0 : wait_on_page_bit_common(): 0.00 : ffff8000101d5674: bl ffff8000101d0650 : ret = -EINTR; 0.00 : ffff8000101d5678: mov w19, #0xfffffffc // #-4 0.00 : ffff8000101d567c: b ffff8000101d4e70 : put_page(): : __put_page(page); 0.00 : ffff8000101d5680: mov x0, x22 : generic_file_buffered_read(): : end_index = (isize - 1) >> PAGE_SHIFT; 0.00 : ffff8000101d5684: str x1, [x29, #152] : put_page(): 0.00 : ffff8000101d5688: bl ffff8000101dfec0 <__put_page> 0.00 : ffff8000101d568c: b ffff8000101d4ea0 : __ll_sc_atomic_sub_return(): 0.00 : ffff8000101d5690: mov w2, #0x1 // #1 0.00 : ffff8000101d5694: b ffff8000101d7384 0.00 : ffff8000101d5698: b ffff8000101d5594 Percent | Source code & Disassembly of vmlinux for cycles (46 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017a460 : : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 6.59 : ffff80001017a460: mrs x1, tpidr_el1 : tick_check_broadcast_expired(): : * to avoid a deep idle transition as we are about to get the : * broadcast IPI right away. : */ : int tick_check_broadcast_expired(void) : { : return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask); 2.29 : ffff80001017a464: adrp x0, ffff80001151d000 0.00 : ffff80001017a468: add x0, x0, #0x18 0.00 : ffff80001017a46c: ldr w0, [x0, x1] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001017a470: adrp x2, ffff800011b22000 0.00 : ffff80001017a474: add x2, x2, #0x3d8 0.00 : ffff80001017a478: cmp w0, #0x0 88.87 : ffff80001017a47c: add w1, w0, #0x3f 0.00 : ffff80001017a480: csel w1, w1, w0, lt // lt = tstop 0.00 : ffff80001017a484: add x2, x2, #0x90 0.00 : ffff80001017a488: asr w1, w1, #6 2.26 : ffff80001017a48c: sxtw x1, w1 0.00 : ffff80001017a490: ldr x1, [x2, x1, lsl #3] 0.00 : ffff80001017a494: lsr x0, x1, x0 : tick_check_broadcast_expired(): : } 0.00 : ffff80001017a498: and w0, w0, #0x1 0.00 : ffff80001017a49c: ret Percent | Source code & Disassembly of vmlinux for cycles (115 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001046cbc0 <__blk_mq_sched_bio_merge>: : __blk_mq_sched_bio_merge(): : return false; : } : : bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio, : unsigned int nr_segs) : { 4.37 : ffff80001046cbc0: stp x29, x30, [sp, #-64]! : blk_mq_get_ctx(): : * care about preemption, since we know the ctx's are persistent. This does : * mean that we can't rely on ctx always matching the currently running CPU. : */ : static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q) : { : return __blk_mq_get_ctx(q, raw_smp_processor_id()); 0.00 : ffff80001046cbc4: adrp x3, ffff80001151d000 0.00 : ffff80001046cbc8: add x3, x3, #0x18 : __blk_mq_get_ctx(): : return per_cpu_ptr(q->queue_ctx, cpu); 0.00 : ffff80001046cbcc: adrp x5, ffff800011909000 : __blk_mq_sched_bio_merge(): 2.62 : ffff80001046cbd0: mov x29, sp 0.00 : ffff80001046cbd4: stp x19, x20, [sp, #16] 0.00 : ffff80001046cbd8: mov x20, x0 0.00 : ffff80001046cbdc: stp x21, x22, [sp, #32] : blk_mq_get_ctx(): : return __blk_mq_get_ctx(q, raw_smp_processor_id()); 0.00 : ffff80001046cbe0: mov x0, x3 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.88 : ffff80001046cbe4: mrs x3, tpidr_el1 : __blk_mq_get_ctx(): : return per_cpu_ptr(q->queue_ctx, cpu); 0.00 : ffff80001046cbe8: ldr w8, [x0, x3] 0.00 : ffff80001046cbec: add x5, x5, #0x928 : __blk_mq_sched_bio_merge(): : struct elevator_queue *e = q->elevator; : struct blk_mq_ctx *ctx = blk_mq_get_ctx(q); : struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, bio->bi_opf, ctx); 6.95 : ffff80001046cbf0: ldr w6, [x1, #16] 0.00 : ffff80001046cbf4: mov w4, #0x2 // #2 : struct elevator_queue *e = q->elevator; 0.00 : ffff80001046cbf8: ldr x7, [x20, #8] : __blk_mq_get_ctx(): 0.90 : ffff80001046cbfc: ldr x0, [x20, #56] 0.00 : ffff80001046cc00: ldr x19, [x5, x8, lsl #3] 0.00 : ffff80001046cc04: add x19, x0, x19 : blk_mq_map_queue(): : if (flags & REQ_HIPRI) 0.00 : ffff80001046cc08: tbnz w6, #25, ffff80001046cc18 <__blk_mq_sched_bio_merge+0x58> : else if ((flags & REQ_OP_MASK) == REQ_OP_READ) 9.48 : ffff80001046cc0c: and w6, w6, #0xff 0.00 : ffff80001046cc10: cmp w6, #0x0 0.00 : ffff80001046cc14: cset w4, eq // eq = none : return ctx->hctxs[type]; 0.00 : ffff80001046cc18: ubfiz x4, x4, #3, #2 0.00 : ffff80001046cc1c: add x4, x19, x4 0.00 : ffff80001046cc20: ldr x21, [x4, #80] : __blk_mq_sched_bio_merge(): : bool ret = false; : enum hctx_type type; : : if (e && e->type->ops.bio_merge) 0.00 : ffff80001046cc24: cbz x7, ffff80001046cc54 <__blk_mq_sched_bio_merge+0x94> 0.00 : ffff80001046cc28: ldr x0, [x7] 0.00 : ffff80001046cc2c: ldr x4, [x0, #56] 0.00 : ffff80001046cc30: cbz x4, ffff80001046cc54 <__blk_mq_sched_bio_merge+0x94> : return e->type->ops.bio_merge(hctx, bio, nr_segs); 0.00 : ffff80001046cc34: mov x0, x21 0.00 : ffff80001046cc38: blr x4 0.00 : ffff80001046cc3c: and w22, w0, #0xff : ret = blk_mq_attempt_merge(q, hctx, ctx, bio, nr_segs); : spin_unlock(&ctx->lock); : } : : return ret; : } 1.77 : ffff80001046cc40: mov w0, w22 0.00 : ffff80001046cc44: ldp x19, x20, [sp, #16] 0.00 : ffff80001046cc48: ldp x21, x22, [sp, #32] 0.00 : ffff80001046cc4c: ldp x29, x30, [sp], #64 0.00 : ffff80001046cc50: ret : if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) && 37.37 : ffff80001046cc54: ldr x0, [x21, #192] : bool ret = false; 0.00 : ffff80001046cc58: mov w22, #0x0 // #0 : if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) && 0.00 : ffff80001046cc5c: tbz w0, #0, ffff80001046cc40 <__blk_mq_sched_bio_merge+0x80> 18.47 : ffff80001046cc60: ldrh w0, [x21, #268] 0.00 : ffff80001046cc64: lsl x0, x0, #4 0.00 : ffff80001046cc68: add x3, x19, x0 : !list_empty_careful(&ctx->rq_lists[type])) { 0.00 : ffff80001046cc6c: add x0, x0, #0x8 0.87 : ffff80001046cc70: add x0, x19, x0 : list_empty_careful(): : * if another CPU could re-list_add() it. : */ : static inline int list_empty_careful(const struct list_head *head) : { : struct list_head *next = head->next; : return (next == head) && (next == head->prev); 0.00 : ffff80001046cc74: ldr x4, [x3, #8] 0.00 : ffff80001046cc78: cmp x0, x4 0.00 : ffff80001046cc7c: b.eq ffff80001046ccf0 <__blk_mq_sched_bio_merge+0x130> // b.none : spin_lock(): : raw_spin_lock_init(&(_lock)->rlock); \ : } while (0) : : static __always_inline void spin_lock(spinlock_t *lock) : { : raw_spin_lock(&lock->rlock); 0.00 : ffff80001046cc80: mov x0, x19 0.00 : ffff80001046cc84: str w2, [x29, #52] 0.00 : ffff80001046cc88: str x1, [x29, #56] 0.00 : ffff80001046cc8c: bl ffff800010cd80f0 <_raw_spin_lock> : blk_mq_attempt_merge(): : if (blk_mq_bio_list_merge(q, &ctx->rq_lists[type], bio, nr_segs)) { 0.00 : ffff80001046cc90: ldrh w4, [x21, #268] 0.00 : ffff80001046cc94: mov x5, #0x8 // #8 0.00 : ffff80001046cc98: ldr w2, [x29, #52] 0.00 : ffff80001046cc9c: mov x0, x20 0.00 : ffff80001046cca0: ldr x1, [x29, #56] : return false; 0.00 : ffff80001046cca4: mov w22, #0x0 // #0 : if (blk_mq_bio_list_merge(q, &ctx->rq_lists[type], bio, nr_segs)) { 0.00 : ffff80001046cca8: mov w3, w2 0.00 : ffff80001046ccac: mov x2, x1 0.00 : ffff80001046ccb0: add x1, x5, w4, uxtw #4 0.00 : ffff80001046ccb4: add x1, x19, x1 0.00 : ffff80001046ccb8: bl ffff80001046c640 0.00 : ffff80001046ccbc: tst w0, #0xff 0.00 : ffff80001046ccc0: b.eq ffff80001046ccd4 <__blk_mq_sched_bio_merge+0x114> // b.none : ctx->rq_merged++; 0.00 : ffff80001046ccc4: ldr x0, [x19, #120] : return true; 0.00 : ffff80001046ccc8: mov w22, #0x1 // #1 : ctx->rq_merged++; 0.00 : ffff80001046cccc: add x0, x0, #0x1 0.00 : ffff80001046ccd0: str x0, [x19, #120] : spin_unlock(): : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ : } while (0) : : static __always_inline void spin_unlock(spinlock_t *lock) : { : raw_spin_unlock(&lock->rlock); 0.00 : ffff80001046ccd4: mov x0, x19 0.00 : ffff80001046ccd8: bl ffff800010cd7c38 <_raw_spin_unlock> : __blk_mq_sched_bio_merge(): : } 0.00 : ffff80001046ccdc: mov w0, w22 0.00 : ffff80001046cce0: ldp x19, x20, [sp, #16] 0.00 : ffff80001046cce4: ldp x21, x22, [sp, #32] 0.00 : ffff80001046cce8: ldp x29, x30, [sp], #64 0.00 : ffff80001046ccec: ret : list_empty_careful(): 12.89 : ffff80001046ccf0: ldr x3, [x3, #16] 0.00 : ffff80001046ccf4: cmp x0, x3 0.00 : ffff80001046ccf8: b.ne ffff80001046cc80 <__blk_mq_sched_bio_merge+0xc0> // b.any 3.43 : ffff80001046ccfc: b ffff80001046cc40 <__blk_mq_sched_bio_merge+0x80> Percent | Source code & Disassembly of vmlinux for cycles (78 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011e940 : : update_load_avg(): : #define SKIP_AGE_LOAD 0x2 : #define DO_ATTACH 0x4 : : /* Update task and its cfs_rq load average */ : static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) : { 3.90 : ffff80001011e940: stp x29, x30, [sp, #-96]! 0.00 : ffff80001011e944: mov x29, sp 6.32 : ffff80001011e948: stp x19, x20, [sp, #16] 0.00 : ffff80001011e94c: mov x19, x0 4.74 : ffff80001011e950: stp x21, x22, [sp, #32] 0.00 : ffff80001011e954: mov x20, x1 1.25 : ffff80001011e958: str x24, [sp, #56] 0.00 : ffff80001011e95c: mov w22, w2 : cfs_rq_clock_pelt(): : return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; : } : #else : static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) : { : return rq_clock_pelt(rq_of(cfs_rq)); 1.30 : ffff80001011e960: ldr x0, [x0, #304] : update_load_avg(): : : /* : * Track task load average for carrying it to new CPU after migrated, and : * track group sched_entity load average for task_h_load calc in migration : */ : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 6.38 : ffff80001011e964: ldr x1, [x1, #192] : rq_clock_pelt(): : return rq->clock_pelt - rq->lost_idle_time; 1.30 : ffff80001011e968: ldr x21, [x0, #2440] 7.70 : ffff80001011e96c: ldr x0, [x0, #2448] 0.00 : ffff80001011e970: sub x21, x21, x0 : update_load_avg(): 0.00 : ffff80001011e974: cbz x1, ffff80001011e988 : __update_load_avg_se(now, cfs_rq, se); 1.22 : ffff80001011e978: mov x2, x20 0.00 : ffff80001011e97c: mov x1, x19 0.00 : ffff80001011e980: mov x0, x21 0.00 : ffff80001011e984: bl ffff800010134868 <__update_load_avg_se> : update_cfs_rq_load_avg(): : if (cfs_rq->removed.nr) { 2.62 : ffff80001011e988: ldr w24, [x19, #196] 0.00 : ffff80001011e98c: cbz w24, ffff80001011ea48 0.00 : ffff80001011e990: str x23, [x29, #48] : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff80001011e994: mov w2, #0xb67e // #46718 0.00 : ffff80001011e998: stp x25, x26, [x29, #64] : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff80001011e99c: add x24, x19, #0xc0 0.00 : ffff80001011e9a0: str x27, [x29, #80] 0.00 : ffff80001011e9a4: mov x0, x24 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff80001011e9a8: ldr w1, [x19, #156] 0.00 : ffff80001011e9ac: add w23, w1, w2 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff80001011e9b0: bl ffff800010cd80f0 <_raw_spin_lock> : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff80001011e9b4: ldr x27, [x19, #216] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff80001011e9b8: ldp x26, x25, [x19, #200] : cfs_rq->removed.nr = 0; 0.00 : ffff80001011e9bc: str wzr, [x19, #196] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff80001011e9c0: stp xzr, xzr, [x19, #200] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff80001011e9c4: mov x0, x24 : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff80001011e9c8: str xzr, [x19, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff80001011e9cc: bl ffff800010cd7c38 <_raw_spin_unlock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011e9d0: ldr x4, [x19, #160] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff80001011e9d4: mov x1, #0x1 // #1 0.00 : ffff80001011e9d8: str x1, [x19, #264] : update_cfs_rq_load_avg(): : decayed = 1; 0.00 : ffff80001011e9dc: mov w24, w1 : sub_positive(&sa->load_avg, r); 0.00 : ffff80001011e9e0: sub x2, x4, x26 : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff80001011e9e4: mov w0, w23 : sub_positive(&sa->load_avg, r); 0.00 : ffff80001011e9e8: cmp x4, x2 : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff80001011e9ec: ldr x3, [x19, #272] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff80001011e9f0: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011e9f4: str x2, [x19, #160] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff80001011e9f8: sub x2, x3, x27 0.00 : ffff80001011e9fc: str x2, [x19, #272] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011ea00: ldr x1, [x19, #136] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff80001011ea04: msub x0, x0, x26, x1 0.00 : ffff80001011ea08: cmp x1, x0 0.00 : ffff80001011ea0c: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011ea10: str x0, [x19, #136] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011ea14: ldr x1, [x19, #176] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_avg, r); 0.00 : ffff80001011ea18: sub x0, x1, x25 0.00 : ffff80001011ea1c: cmp x1, x0 0.00 : ffff80001011ea20: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011ea24: str x0, [x19, #176] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011ea28: ldr w0, [x19, #152] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_sum, r * divider); 0.00 : ffff80001011ea2c: msub w1, w23, w25, w0 0.00 : ffff80001011ea30: cmp w0, w1 0.00 : ffff80001011ea34: csel w1, w1, wzr, cs // cs = hs, nlast : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001011ea38: str w1, [x19, #152] 0.00 : ffff80001011ea3c: ldr x23, [x29, #48] 0.00 : ffff80001011ea40: ldp x25, x26, [x29, #64] 0.00 : ffff80001011ea44: ldr x27, [x29, #80] : update_cfs_rq_load_avg(): : decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 10.41 : ffff80001011ea48: mov x1, x19 0.00 : ffff80001011ea4c: mov x0, x21 0.00 : ffff80001011ea50: bl ffff800010134a68 <__update_load_avg_cfs_rq> 6.41 : ffff80001011ea54: orr w0, w0, w24 : propagate_entity_load_avg(): : if (entity_is_task(se)) 2.58 : ffff80001011ea58: ldr x1, [x20, #136] 0.00 : ffff80001011ea5c: cbz x1, ffff80001011ebd4 : if (!gcfs_rq->propagate) 1.30 : ffff80001011ea60: ldr x2, [x1, #264] 0.00 : ffff80001011ea64: cbz x2, ffff80001011ebd4 : gcfs_rq->propagate = 0; 0.00 : ffff80001011ea68: str xzr, [x1, #264] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff80001011ea6c: mov x5, #0x1 // #1 : propagate_entity_load_avg(): : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 0.00 : ffff80001011ea70: ldr x4, [x1, #272] : cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011ea74: ldr x3, [x20, #128] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff80001011ea78: ldr x2, [x3, #272] 0.00 : ffff80001011ea7c: add x2, x2, x4 0.00 : ffff80001011ea80: stp x5, x2, [x3, #264] : propagate_entity_load_avg(): : update_tg_cfs_util(cfs_rq, se, gcfs_rq); 0.00 : ffff80001011ea84: ldr x4, [x1, #176] : update_tg_cfs_util(): : long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 0.00 : ffff80001011ea88: ldr x2, [x20, #240] 0.00 : ffff80001011ea8c: sub x2, x4, x2 : if (!delta) 0.00 : ffff80001011ea90: cmp x2, #0x0 0.00 : ffff80001011ea94: b.ne ffff80001011ec50 // b.any : update_tg_cfs_runnable(): : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 0.00 : ffff80001011ea98: ldr x2, [x1, #272] 0.00 : ffff80001011ea9c: orr w0, w0, #0x1 : if (!runnable_sum) 0.00 : ffff80001011eaa0: cmp x2, #0x0 0.00 : ffff80001011eaa4: b.eq ffff80001011ebb0 // b.none : gcfs_rq->prop_runnable_sum = 0; 0.00 : ffff80001011eaa8: str xzr, [x1, #272] : if (runnable_sum >= 0) { 0.00 : ffff80001011eaac: b.lt ffff80001011eccc // b.tstop : runnable_sum += se->avg.load_sum; 0.00 : ffff80001011eab0: ldr x1, [x20, #200] : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff80001011eab4: mov x4, #0xba7e // #47742 : runnable_sum += se->avg.load_sum; 0.00 : ffff80001011eab8: add x2, x2, x1 : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff80001011eabc: cmp x2, x4 0.00 : ffff80001011eac0: csel x2, x2, x4, le : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff80001011eac4: ldr w5, [x20, #216] : div_s64_rem(): : * Return: sets ``*remainder``, then returns dividend / divisor : */ : static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) : { : *remainder = dividend % divisor; : return dividend / divisor; 0.00 : ffff80001011eac8: mov x6, #0x5041 // #20545 : se_weight(): : /* : * XXX we want to get rid of these helpers and use the full load resolution. : */ : static inline long se_weight(struct sched_entity *se) : { : return scale_load_down(se->load.weight); 0.00 : ffff80001011eacc: ldr x4, [x20] : div_s64_rem(): 0.00 : ffff80001011ead0: movk x6, #0xafb, lsl #16 0.00 : ffff80001011ead4: movk x6, #0x400, lsl #32 : update_tg_cfs_runnable(): : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff80001011ead8: ldr x7, [x20, #224] : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff80001011eadc: lsr w5, w5, #10 : div_s64_rem(): 0.00 : ffff80001011eae0: movk x6, #0xafb5, lsl #48 : update_tg_cfs_runnable(): : runnable_sum = max(runnable_sum, running_sum); 0.00 : ffff80001011eae4: cmp x5, x2 : se_weight(): 0.00 : ffff80001011eae8: lsr x4, x4, #10 : update_tg_cfs_runnable(): 0.00 : ffff80001011eaec: csel x5, x5, x2, ge // ge = tcont : se->avg.load_sum = runnable_sum; 0.00 : ffff80001011eaf0: str x5, [x20, #200] : load_sum = (s64)se_weight(se) * runnable_sum; 0.00 : ffff80001011eaf4: mul x2, x4, x5 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 0.00 : ffff80001011eaf8: msub x4, x4, x1, x2 : div_s64_rem(): 0.00 : ffff80001011eafc: lsr x1, x2, #1 0.00 : ffff80001011eb00: umulh x1, x1, x6 0.00 : ffff80001011eb04: lsr x1, x1, #14 : update_tg_cfs_runnable(): : se->avg.load_avg = load_avg; 0.00 : ffff80001011eb08: str x1, [x20, #224] : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff80001011eb0c: sub x1, x1, x7 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011eb10: ldr x2, [x3, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_avg, delta_avg); 0.00 : ffff80001011eb14: cmp x1, #0x0 0.00 : ffff80001011eb18: add x1, x1, x2 0.00 : ffff80001011eb1c: ccmp x2, x1, #0x2, lt // lt = tstop 0.00 : ffff80001011eb20: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011eb24: str x1, [x3, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_sum, delta_sum); 0.00 : ffff80001011eb28: cmp x4, #0x0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011eb2c: ldr x2, [x3, #136] : update_tg_cfs_runnable(): 0.00 : ffff80001011eb30: add x1, x4, x2 0.00 : ffff80001011eb34: ccmp x2, x1, #0x2, lt // lt = tstop 0.00 : ffff80001011eb38: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011eb3c: str x1, [x3, #136] : se_runnable(): : } : : static inline long se_runnable(struct sched_entity *se) : { : return scale_load_down(se->runnable_weight); 0.00 : ffff80001011eb40: ldr x2, [x20, #16] : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff80001011eb44: ldr w4, [x20, #64] : se_runnable(): 0.00 : ffff80001011eb48: lsr x2, x2, #10 : update_tg_cfs_runnable(): : runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 0.00 : ffff80001011eb4c: mul x2, x2, x5 : div_s64_rem(): 0.00 : ffff80001011eb50: lsr x1, x2, #1 0.00 : ffff80001011eb54: umulh x1, x1, x6 0.00 : ffff80001011eb58: lsr x1, x1, #14 : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff80001011eb5c: cbz w4, ffff80001011eba8 : delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 0.00 : ffff80001011eb60: ldr x7, [x20, #232] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011eb64: ldr x8, [x3, #168] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff80001011eb68: ldr x4, [x20] : update_tg_cfs_runnable(): 0.00 : ffff80001011eb6c: sub x7, x1, x7 : se_weight(se) * se->avg.runnable_load_sum; 0.00 : ffff80001011eb70: ldr x9, [x20, #208] : add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 0.00 : ffff80001011eb74: add x6, x7, x8 0.00 : ffff80001011eb78: cmp x7, #0x0 0.00 : ffff80001011eb7c: ccmp x8, x6, #0x2, lt // lt = tstop : se_weight(): 0.00 : ffff80001011eb80: lsr x4, x4, #10 : update_tg_cfs_runnable(): 0.00 : ffff80001011eb84: csel x6, x6, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011eb88: str x6, [x3, #168] : update_tg_cfs_runnable(): : delta_sum = runnable_load_sum - 0.00 : ffff80001011eb8c: msub x2, x4, x9, x2 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011eb90: ldr x4, [x3, #144] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 0.00 : ffff80001011eb94: cmp x2, #0x0 0.00 : ffff80001011eb98: add x2, x2, x4 0.00 : ffff80001011eb9c: ccmp x4, x2, #0x2, lt // lt = tstop 0.00 : ffff80001011eba0: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011eba4: str x2, [x3, #144] : update_tg_cfs_runnable(): : se->avg.runnable_load_sum = runnable_sum; 0.00 : ffff80001011eba8: str x5, [x20, #208] : se->avg.runnable_load_avg = runnable_load_avg; 0.00 : ffff80001011ebac: str x1, [x20, #232] : update_load_avg(): : : decayed = update_cfs_rq_load_avg(now, cfs_rq); : decayed |= propagate_entity_load_avg(se); : : if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 0.00 : ffff80001011ebb0: ldr x1, [x20, #192] 0.00 : ffff80001011ebb4: cbnz x1, ffff80001011ebe4 0.00 : ffff80001011ebb8: tbnz w22, #2, ffff80001011ec84 : * IOW we're enqueueing a task on a new CPU. : */ : attach_entity_load_avg(cfs_rq, se); : update_tg_load_avg(cfs_rq, 0); : : } else if (decayed) { 8.97 : ffff80001011ebbc: cbnz w0, ffff80001011ebe4 : cfs_rq_util_change(cfs_rq, 0); : : if (flags & UPDATE_TG) : update_tg_load_avg(cfs_rq, 0); : } : } 5.14 : ffff80001011ebc0: ldp x19, x20, [sp, #16] 18.08 : ffff80001011ebc4: ldp x21, x22, [sp, #32] 0.00 : ffff80001011ebc8: ldr x24, [sp, #56] 0.00 : ffff80001011ebcc: ldp x29, x30, [sp], #96 0.00 : ffff80001011ebd0: ret : if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 9.09 : ffff80001011ebd4: ldr x1, [x20, #192] 0.00 : ffff80001011ebd8: cbnz x1, ffff80001011ebbc 0.00 : ffff80001011ebdc: tbz w22, #2, ffff80001011ebbc 0.00 : ffff80001011ebe0: b ffff80001011ec84 : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff80001011ebe4: ldr x1, [x19, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff80001011ebe8: add x0, x1, #0x80 0.00 : ffff80001011ebec: cmp x19, x0 0.00 : ffff80001011ebf0: b.eq ffff80001011ec94 // b.none : update_load_avg(): : if (flags & UPDATE_TG) 0.00 : ffff80001011ebf4: tbz w22, #0, ffff80001011ebc0 : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff80001011ebf8: ldr x1, [x19, #336] 0.00 : ffff80001011ebfc: adrp x0, ffff800011afb000 0.00 : ffff80001011ec00: add x0, x0, #0xd80 0.00 : ffff80001011ec04: cmp x1, x0 0.00 : ffff80001011ec08: b.eq ffff80001011ebc0 // b.none : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 0.00 : ffff80001011ec0c: ldr x0, [x19, #160] 0.00 : ffff80001011ec10: ldr x2, [x19, #256] : if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 0.00 : ffff80001011ec14: subs x0, x0, x2 0.00 : ffff80001011ec18: cneg x3, x0, mi // mi = first 0.00 : ffff80001011ec1c: cmp x3, x2, lsr #6 0.00 : ffff80001011ec20: b.ls ffff80001011ebc0 // b.plast : atomic_long_add(delta, &cfs_rq->tg->load_avg); 0.00 : ffff80001011ec24: add x2, x1, #0x140 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011ec28: b ffff80001011ecf8 0.00 : ffff80001011ec2c: b ffff80001011ecf8 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff80001011ec30: stadd x0, [x2] : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff80001011ec34: ldr x0, [x19, #160] 0.00 : ffff80001011ec38: str x0, [x19, #256] : update_load_avg(): : } 0.00 : ffff80001011ec3c: ldp x19, x20, [sp, #16] 0.00 : ffff80001011ec40: ldp x21, x22, [sp, #32] 0.00 : ffff80001011ec44: ldr x24, [sp, #56] 0.00 : ffff80001011ec48: ldp x29, x30, [sp], #96 0.00 : ffff80001011ec4c: ret : update_tg_cfs_util(): : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff80001011ec50: mov w5, #0xba7e // #47742 : se->avg.util_avg = gcfs_rq->avg.util_avg; 0.00 : ffff80001011ec54: str x4, [x20, #240] : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff80001011ec58: mul w4, w4, w5 0.00 : ffff80001011ec5c: str w4, [x20, #216] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011ec60: ldr x4, [x3, #176] : update_tg_cfs_util(): : add_positive(&cfs_rq->avg.util_avg, delta); 0.00 : ffff80001011ec64: add x2, x2, x4 0.00 : ffff80001011ec68: ccmp x4, x2, #0x2, lt // lt = tstop 0.00 : ffff80001011ec6c: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011ec70: str x2, [x3, #176] : update_tg_cfs_util(): : cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff80001011ec74: ldr x2, [x3, #176] 0.00 : ffff80001011ec78: mul w5, w5, w2 0.00 : ffff80001011ec7c: str w5, [x3, #152] 0.00 : ffff80001011ec80: b ffff80001011ea98 : update_load_avg(): : attach_entity_load_avg(cfs_rq, se); 0.00 : ffff80001011ec84: mov x1, x20 0.00 : ffff80001011ec88: mov x0, x19 0.00 : ffff80001011ec8c: bl ffff80001011c660 0.00 : ffff80001011ec90: b ffff80001011ebf8 : cpufreq_update_util(): : */ : static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) : { : struct update_util_data *data; : : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff80001011ec94: ldrsw x3, [x1, #2568] 0.00 : ffff80001011ec98: adrp x2, ffff800011909000 0.00 : ffff80001011ec9c: add x2, x2, #0x928 0.00 : ffff80001011eca0: adrp x0, ffff80001151f000 0.00 : ffff80001011eca4: add x0, x0, #0x158 0.00 : ffff80001011eca8: ldr x2, [x2, x3, lsl #3] : __read_once_size(): : __READ_ONCE_SIZE; 1.31 : ffff80001011ecac: ldr x0, [x0, x2] : cpufreq_update_util(): : cpu_of(rq))); : if (data) 0.00 : ffff80001011ecb0: cbz x0, ffff80001011ebf4 : data->func(data, rq_clock(rq), flags); 0.00 : ffff80001011ecb4: ldr x3, [x0] 0.00 : ffff80001011ecb8: mov w2, #0x0 // #0 0.00 : ffff80001011ecbc: ldr x1, [x1, #2400] 0.00 : ffff80001011ecc0: blr x3 : update_load_avg(): : if (flags & UPDATE_TG) 0.00 : ffff80001011ecc4: tbz w22, #0, ffff80001011ebc0 0.00 : ffff80001011ecc8: b ffff80001011ebf8 : update_tg_cfs_runnable(): : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff80001011eccc: ldr x4, [x1] : u64 runnable_load_sum, load_sum = 0; 0.00 : ffff80001011ecd0: mov x2, #0x0 // #0 : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff80001011ecd4: lsr x4, x4, #10 0.00 : ffff80001011ecd8: cbz x4, ffff80001011ece8 : div_s64_rem(): 0.00 : ffff80001011ecdc: ldr x2, [x1, #136] : *remainder = dividend % divisor; 0.00 : ffff80001011ece0: sxtw x4, w4 : return dividend / divisor; 0.00 : ffff80001011ece4: sdiv x2, x2, x4 : update_tg_cfs_runnable(): : runnable_sum = min(se->avg.load_sum, load_sum); 0.00 : ffff80001011ece8: ldr x1, [x20, #200] 0.00 : ffff80001011ecec: cmp x1, x2 0.00 : ffff80001011ecf0: csel x2, x1, x2, ls // ls = plast 0.00 : ffff80001011ecf4: b ffff80001011eac4 : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff80001011ecf8: add x1, x1, #0x140 0.00 : ffff80001011ecfc: b ffff800010128974 0.00 : ffff80001011ed00: b ffff80001011ec34 Percent | Source code & Disassembly of vmlinux for cycles (97 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104674e0 <__blk_mq_try_issue_directly>: : __blk_mq_try_issue_directly(): : : static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, : struct request *rq, : blk_qc_t *cookie, : bool bypass_insert, bool last) : { 0.00 : ffff8000104674e0: stp x29, x30, [sp, #-96]! 0.00 : ffff8000104674e4: mov x29, sp 4.17 : ffff8000104674e8: stp x19, x20, [sp, #16] 0.00 : ffff8000104674ec: adrp x19, ffff800011909000 2.09 : ffff8000104674f0: str x22, [sp, #40] 0.00 : ffff8000104674f4: add x5, x19, #0x908 0.00 : ffff8000104674f8: mov x20, x1 1.04 : ffff8000104674fc: ldr x6, [x5] 7.07 : ffff800010467500: str x6, [x29, #88] 0.00 : ffff800010467504: mov x6, #0x0 // #0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 4.17 : ffff800010467508: ldr x1, [x0, #24] : __blk_mq_try_issue_directly(): : * : * When queue is stopped or quiesced, ignore 'bypass_insert' from : * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller, : * and avoid driver to try to dispatch again. : */ : if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) { 0.00 : ffff80001046750c: tbnz w1, #0, ffff8000104675ec <__blk_mq_try_issue_directly+0x10c> 4.22 : ffff800010467510: str x23, [x29, #48] 0.00 : ffff800010467514: and w23, w3, #0xff : struct request_queue *q = rq->q; 1.03 : ffff800010467518: ldr x1, [x20] : test_bit(): 0.00 : ffff80001046751c: ldr x3, [x1, #104] : __blk_mq_try_issue_directly(): : if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) { 0.00 : ffff800010467520: tst w3, #0x1000000 0.00 : ffff800010467524: b.eq ffff800010467568 <__blk_mq_try_issue_directly+0x88> // b.none : run_queue = false; 0.00 : ffff800010467528: ldr x23, [x29, #48] 0.00 : ffff80001046752c: mov w2, #0x0 // #0 : insert: : if (bypass_insert) : return BLK_STS_RESOURCE; : : blk_mq_request_bypass_insert(rq, false, run_queue); : return BLK_STS_OK; 0.00 : ffff800010467530: mov w22, #0x0 // #0 : blk_mq_request_bypass_insert(rq, false, run_queue); 0.00 : ffff800010467534: mov w1, #0x0 // #0 0.00 : ffff800010467538: mov x0, x20 0.00 : ffff80001046753c: bl ffff800010467460 : } 4.17 : ffff800010467540: add x19, x19, #0x908 0.00 : ffff800010467544: mov w0, w22 0.00 : ffff800010467548: ldr x2, [x29, #88] 0.00 : ffff80001046754c: ldr x1, [x19] 0.00 : ffff800010467550: eor x1, x2, x1 0.00 : ffff800010467554: cbnz x1, ffff8000104676c0 <__blk_mq_try_issue_directly+0x1e0> 1.00 : ffff800010467558: ldp x19, x20, [sp, #16] 1.04 : ffff80001046755c: ldr x22, [sp, #40] 0.00 : ffff800010467560: ldp x29, x30, [sp], #96 0.00 : ffff800010467564: ret : if (q->elevator && !bypass_insert) 2.09 : ffff800010467568: ldr x1, [x1, #8] 0.00 : ffff80001046756c: cmp x1, #0x0 0.00 : ffff800010467570: ccmp w23, #0x0, #0x0, ne // ne = any 0.00 : ffff800010467574: b.eq ffff8000104675e0 <__blk_mq_try_issue_directly+0x100> // b.none 15.43 : ffff800010467578: str x21, [x29, #32] 0.00 : ffff80001046757c: and w22, w4, #0xff 7.32 : ffff800010467580: str x24, [x29, #56] 0.00 : ffff800010467584: mov x21, x0 0.00 : ffff800010467588: mov x24, x2 : blk_mq_get_dispatch_budget(): : : static inline bool blk_mq_get_dispatch_budget(struct blk_mq_hw_ctx *hctx) : { : struct request_queue *q = hctx->queue; : : if (q->mq_ops->get_budget) 2.06 : ffff80001046758c: ldr x1, [x0, #208] 0.00 : ffff800010467590: ldr x1, [x1, #48] 1.04 : ffff800010467594: ldr x1, [x1, #16] 0.00 : ffff800010467598: cbz x1, ffff8000104675a8 <__blk_mq_try_issue_directly+0xc8> : return q->mq_ops->get_budget(hctx); 0.00 : ffff80001046759c: blr x1 : __blk_mq_try_issue_directly(): : if (!blk_mq_get_dispatch_budget(hctx)) 0.00 : ffff8000104675a0: tst w0, #0xff 0.00 : ffff8000104675a4: b.eq ffff8000104675d0 <__blk_mq_try_issue_directly+0xf0> // b.none : if (!blk_mq_get_driver_tag(rq)) { 13.36 : ffff8000104675a8: mov x0, x20 0.00 : ffff8000104675ac: bl ffff800010466d00 0.00 : ffff8000104675b0: tst w0, #0xff 0.00 : ffff8000104675b4: b.ne ffff8000104675fc <__blk_mq_try_issue_directly+0x11c> // b.any : blk_mq_put_dispatch_budget(): : if (q->mq_ops->put_budget) 0.00 : ffff8000104675b8: ldr x0, [x21, #208] 0.00 : ffff8000104675bc: ldr x0, [x0, #48] 0.00 : ffff8000104675c0: ldr x1, [x0, #24] 0.00 : ffff8000104675c4: cbz x1, ffff8000104675d0 <__blk_mq_try_issue_directly+0xf0> : q->mq_ops->put_budget(hctx); 0.00 : ffff8000104675c8: mov x0, x21 0.00 : ffff8000104675cc: blr x1 : __blk_mq_try_issue_directly(): : return BLK_STS_RESOURCE; 0.00 : ffff8000104675d0: mov w22, #0x9 // #9 0.00 : ffff8000104675d4: ldr x21, [x29, #32] : if (bypass_insert) 0.00 : ffff8000104675d8: cbnz w23, ffff8000104675f4 <__blk_mq_try_issue_directly+0x114> 0.00 : ffff8000104675dc: ldr x24, [x29, #56] : run_queue = false; 0.00 : ffff8000104675e0: mov w2, #0x1 // #1 0.00 : ffff8000104675e4: ldr x23, [x29, #48] 0.00 : ffff8000104675e8: b ffff800010467530 <__blk_mq_try_issue_directly+0x50> 0.00 : ffff8000104675ec: mov w2, #0x0 // #0 0.00 : ffff8000104675f0: b ffff800010467530 <__blk_mq_try_issue_directly+0x50> 0.00 : ffff8000104675f4: ldp x23, x24, [x29, #48] 0.00 : ffff8000104675f8: b ffff800010467540 <__blk_mq_try_issue_directly+0x60> 0.00 : ffff8000104675fc: ldr w0, [x21, #428] : __blk_mq_issue_directly(): : new_cookie = request_to_qc_t(hctx, rq); 8.30 : ffff800010467600: ldr w1, [x20, #32] : struct blk_mq_queue_data bd = { 6.00 : ffff800010467604: str x20, [x29, #72] 0.00 : ffff800010467608: strb w22, [x29, #80] : request_to_qc_t(): : ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) : : static inline blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, : struct request *rq) : { : if (rq->tag != -1) 0.00 : ffff80001046760c: cmn w1, #0x1 0.00 : ffff800010467610: lsl w0, w0, #16 : __blk_mq_issue_directly(): : struct request_queue *q = rq->q; 0.00 : ffff800010467614: ldr x2, [x20] : request_to_qc_t(): 0.00 : ffff800010467618: b.ne ffff800010467624 <__blk_mq_try_issue_directly+0x144> // b.any : return rq->tag | (hctx->queue_num << BLK_QC_T_SHIFT); : : return rq->internal_tag | (hctx->queue_num << BLK_QC_T_SHIFT) | 0.00 : ffff80001046761c: ldr w1, [x20, #36] 0.00 : ffff800010467620: orr w0, w0, #0x80000000 : __blk_mq_issue_directly(): : ret = q->mq_ops->queue_rq(hctx, &bd); 0.00 : ffff800010467624: ldr x2, [x2, #48] : request_to_qc_t(): 0.00 : ffff800010467628: orr w23, w0, w1 : __blk_mq_issue_directly(): 0.00 : ffff80001046762c: add x1, x29, #0x48 0.00 : ffff800010467630: mov x0, x21 7.25 : ffff800010467634: ldr x2, [x2] 0.00 : ffff800010467638: blr x2 1.04 : ffff80001046763c: and w22, w0, #0xff : blk_mq_update_dispatch_busy(): : if (hctx->queue->elevator) 0.00 : ffff800010467640: ldr x1, [x21, #208] 0.00 : ffff800010467644: add x0, x21, #0x108 : __blk_mq_issue_directly(): : switch (ret) { 0.00 : ffff800010467648: cmp w22, #0x9 : blk_mq_update_dispatch_busy(): : if (hctx->queue->elevator) 1.03 : ffff80001046764c: ldr x1, [x1, #8] : __blk_mq_issue_directly(): : switch (ret) { 0.00 : ffff800010467650: b.eq ffff800010467678 <__blk_mq_try_issue_directly+0x198> // b.none 2.10 : ffff800010467654: cmp w22, #0xd 0.00 : ffff800010467658: b.eq ffff800010467678 <__blk_mq_try_issue_directly+0x198> // b.none 0.00 : ffff80001046765c: cbz w22, ffff800010467690 <__blk_mq_try_issue_directly+0x1b0> : blk_mq_update_dispatch_busy(): : if (hctx->queue->elevator) 0.00 : ffff800010467660: cbz x1, ffff8000104676b8 <__blk_mq_try_issue_directly+0x1d8> : __blk_mq_issue_directly(): : *cookie = BLK_QC_T_NONE; 0.00 : ffff800010467664: mov w0, #0xffffffff // #-1 0.00 : ffff800010467668: str w0, [x24] : __blk_mq_try_issue_directly(): : return __blk_mq_issue_directly(hctx, rq, cookie, last); 0.00 : ffff80001046766c: ldr x21, [x29, #32] 0.00 : ffff800010467670: ldp x23, x24, [x29, #48] 0.00 : ffff800010467674: b ffff800010467540 <__blk_mq_try_issue_directly+0x60> : blk_mq_update_dispatch_busy(): : if (hctx->queue->elevator) 0.00 : ffff800010467678: cbz x1, ffff8000104676a4 <__blk_mq_try_issue_directly+0x1c4> : __blk_mq_issue_directly(): : __blk_mq_requeue_request(rq); 0.00 : ffff80001046767c: mov x0, x20 0.00 : ffff800010467680: bl ffff800010465100 <__blk_mq_requeue_request> 0.00 : ffff800010467684: ldr x21, [x29, #32] 0.00 : ffff800010467688: ldp x23, x24, [x29, #48] 0.00 : ffff80001046768c: b ffff800010467540 <__blk_mq_try_issue_directly+0x60> : blk_mq_update_dispatch_busy(): : if (hctx->queue->elevator) 0.00 : ffff800010467690: cbz x1, ffff8000104676b0 <__blk_mq_try_issue_directly+0x1d0> : __blk_mq_issue_directly(): : *cookie = new_cookie; 0.00 : ffff800010467694: str w23, [x24] 0.00 : ffff800010467698: ldr x21, [x29, #32] 0.00 : ffff80001046769c: ldp x23, x24, [x29, #48] 0.00 : ffff8000104676a0: b ffff800010467540 <__blk_mq_try_issue_directly+0x60> : blk_mq_update_dispatch_busy(): 0.00 : ffff8000104676a4: mov w1, #0x1 // #1 0.00 : ffff8000104676a8: bl ffff800010464740 0.00 : ffff8000104676ac: b ffff80001046767c <__blk_mq_try_issue_directly+0x19c> 1.99 : ffff8000104676b0: bl ffff800010464740 0.96 : ffff8000104676b4: b ffff800010467694 <__blk_mq_try_issue_directly+0x1b4> 0.00 : ffff8000104676b8: bl ffff800010464740 0.00 : ffff8000104676bc: b ffff800010467664 <__blk_mq_try_issue_directly+0x184> 0.00 : ffff8000104676c0: str x21, [x29, #32] 0.00 : ffff8000104676c4: stp x23, x24, [x29, #48] : __blk_mq_try_issue_directly(): : } 0.00 : ffff8000104676c8: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (40 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100ec5f8 : : ksoftirqd_running(): : * right now. Let ksoftirqd handle this at its own rate, to get fairness, : * unless we're doing some of the synchronous softirqs. : */ : #define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ)) : static bool ksoftirqd_running(unsigned long pending) : { 19.29 : ffff8000100ec5f8: stp x29, x30, [sp, #-16]! : struct task_struct *tsk = __this_cpu_read(ksoftirqd); 0.00 : ffff8000100ec5fc: adrp x1, ffff80001151e000 : : if (pending & SOFTIRQ_NOW_MASK) 2.47 : ffff8000100ec600: mov x2, #0x41 // #65 : struct task_struct *tsk = __this_cpu_read(ksoftirqd); 0.00 : ffff8000100ec604: add x1, x1, #0xf80 : { 4.98 : ffff8000100ec608: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000100ec60c: mrs x3, tpidr_el1 : ksoftirqd_running(): : if (pending & SOFTIRQ_NOW_MASK) 0.00 : ffff8000100ec610: tst x0, x2 0.00 : ffff8000100ec614: b.ne ffff8000100ec634 // b.any : struct task_struct *tsk = __this_cpu_read(ksoftirqd); 0.00 : ffff8000100ec618: ldr x1, [x1, x3] 0.00 : ffff8000100ec61c: mov w0, #0x0 // #0 : return false; : return tsk && (tsk->state == TASK_RUNNING) && 0.00 : ffff8000100ec620: cbz x1, ffff8000100ec62c 2.38 : ffff8000100ec624: ldr x2, [x1, #24] 0.00 : ffff8000100ec628: cbz x2, ffff8000100ec640 : !__kthread_should_park(tsk); : } 70.89 : ffff8000100ec62c: ldp x29, x30, [sp], #16 0.00 : ffff8000100ec630: ret : return false; 0.00 : ffff8000100ec634: mov w0, #0x0 // #0 : } 0.00 : ffff8000100ec638: ldp x29, x30, [sp], #16 0.00 : ffff8000100ec63c: ret : !__kthread_should_park(tsk); 0.00 : ffff8000100ec640: mov x0, x1 0.00 : ffff8000100ec644: bl ffff80001010bca0 <__kthread_should_park> 0.00 : ffff8000100ec648: and w0, w0, #0xff : return tsk && (tsk->state == TASK_RUNNING) && 0.00 : ffff8000100ec64c: eor w0, w0, #0x1 : } 0.00 : ffff8000100ec650: ldp x29, x30, [sp], #16 0.00 : ffff8000100ec654: ret Percent | Source code & Disassembly of vmlinux for cycles (111 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104b0720 <__sbitmap_get_word>: : __sbitmap_get_word(): : } : EXPORT_SYMBOL_GPL(sbitmap_resize); : : static int __sbitmap_get_word(unsigned long *word, unsigned long depth, : unsigned int hint, bool wrap) : { 10.71 : ffff8000104b0720: stp x29, x30, [sp, #-96]! 0.00 : ffff8000104b0724: mov w2, w2 0.00 : ffff8000104b0728: mov x29, sp 2.71 : ffff8000104b072c: stp x19, x20, [sp, #16] : test_and_set_bit_lock(): : */ : static inline int test_and_set_bit_lock(unsigned int nr, : volatile unsigned long *p) : { : long old; : unsigned long mask = BIT_MASK(nr); 0.00 : ffff8000104b0730: mov x19, #0x1 // #1 : __sbitmap_get_word(): 0.90 : ffff8000104b0734: stp x21, x22, [sp, #32] 0.00 : ffff8000104b0738: adrp x21, ffff800011909000 0.00 : ffff8000104b073c: stp x23, x24, [sp, #48] : unsigned int orig_hint = hint; 0.89 : ffff8000104b0740: mov w22, w2 : { 1.82 : ffff8000104b0744: stp x25, x26, [sp, #64] 0.00 : ffff8000104b0748: mov x26, x1 0.00 : ffff8000104b074c: add x1, x21, #0x908 0.00 : ffff8000104b0750: mov x24, x0 0.00 : ffff8000104b0754: mov x25, x2 0.00 : ffff8000104b0758: and w23, w3, #0xff 0.00 : ffff8000104b075c: ldr x0, [x1] 0.00 : ffff8000104b0760: str x0, [x29, #88] 0.00 : ffff8000104b0764: mov x0, #0x0 // #0 : : if (!test_and_set_bit_lock(nr, word)) : break; : : hint = nr + 1; : if (hint >= depth - 1) 0.00 : ffff8000104b0768: sub x20, x26, #0x1 0.00 : ffff8000104b076c: nop : nr = find_next_zero_bit(word, depth, hint); 8.20 : ffff8000104b0770: mov x1, x26 0.00 : ffff8000104b0774: mov x0, x24 0.00 : ffff8000104b0778: bl ffff80001048da28 4.56 : ffff8000104b077c: mov w4, w0 : if (unlikely(nr >= depth)) { 3.65 : ffff8000104b0780: cmp x26, w0, sxtw 0.00 : ffff8000104b0784: b.ls ffff8000104b07d0 <__sbitmap_get_word+0xb0> // b.plast : test_and_set_bit_lock(): : : p += BIT_WORD(nr); 0.00 : ffff8000104b0788: lsr w1, w0, #6 : unsigned long mask = BIT_MASK(nr); 0.00 : ffff8000104b078c: lsl x2, x19, x0 : p += BIT_WORD(nr); 0.00 : ffff8000104b0790: add x3, x24, x1, lsl #3 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 9.92 : ffff8000104b0794: ldr x1, [x24, x1, lsl #3] : test_and_set_bit_lock(): : if (READ_ONCE(*p) & mask) 0.00 : ffff8000104b0798: tst x2, x1 0.00 : ffff8000104b079c: b.eq ffff8000104b07e8 <__sbitmap_get_word+0xc8> // b.none : __sbitmap_get_word(): : hint = nr + 1; 0.00 : ffff8000104b07a0: add w2, w0, #0x1 0.00 : ffff8000104b07a4: mov x25, x2 : if (hint >= depth - 1) 0.00 : ffff8000104b07a8: cmp x2, x20 0.00 : ffff8000104b07ac: b.cc ffff8000104b0770 <__sbitmap_get_word+0x50> // b.lo, b.ul, b.last 0.00 : ffff8000104b07b0: mov x2, #0x0 // #0 : nr = find_next_zero_bit(word, depth, hint); 0.00 : ffff8000104b07b4: mov x1, x26 0.00 : ffff8000104b07b8: mov x0, x24 : hint = orig_hint = 0; 0.00 : ffff8000104b07bc: mov w25, #0x0 // #0 : nr = find_next_zero_bit(word, depth, hint); 0.00 : ffff8000104b07c0: bl ffff80001048da28 0.00 : ffff8000104b07c4: mov w4, w0 : if (unlikely(nr >= depth)) { 0.00 : ffff8000104b07c8: cmp x26, w0, sxtw 0.00 : ffff8000104b07cc: b.hi ffff8000104b0788 <__sbitmap_get_word+0x68> // b.pmore : if (orig_hint && hint && wrap) { 0.00 : ffff8000104b07d0: cmp w22, #0x0 0.00 : ffff8000104b07d4: ccmp w25, #0x0, #0x4, ne // ne = any 0.00 : ffff8000104b07d8: ccmp w23, #0x0, #0x4, ne // ne = any 0.00 : ffff8000104b07dc: b.eq ffff8000104b0838 <__sbitmap_get_word+0x118> // b.none : hint = orig_hint = 0; 0.00 : ffff8000104b07e0: mov w22, #0x0 // #0 0.00 : ffff8000104b07e4: b ffff8000104b07b0 <__sbitmap_get_word+0x90> : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 7.26 : ffff8000104b07e8: b ffff8000104b0830 <__sbitmap_get_word+0x110> 0.91 : ffff8000104b07ec: b ffff8000104b0830 <__sbitmap_get_word+0x110> : __lse_atomic64_fetch_or_acquire(): : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff8000104b07f0: mov x1, x2 0.00 : ffff8000104b07f4: ldseta x1, x1, [x3] : __sbitmap_get_word(): : if (!test_and_set_bit_lock(nr, word)) 42.18 : ffff8000104b07f8: tst x1, x2 0.00 : ffff8000104b07fc: b.ne ffff8000104b07a0 <__sbitmap_get_word+0x80> // b.any : hint = 0; : } : : return nr; : } 0.00 : ffff8000104b0800: add x21, x21, #0x908 0.00 : ffff8000104b0804: mov w0, w4 1.81 : ffff8000104b0808: ldr x2, [x29, #88] 0.91 : ffff8000104b080c: ldr x1, [x21] 0.00 : ffff8000104b0810: eor x1, x2, x1 0.00 : ffff8000104b0814: cbnz x1, ffff8000104b0840 <__sbitmap_get_word+0x120> 0.00 : ffff8000104b0818: ldp x19, x20, [sp, #16] 2.73 : ffff8000104b081c: ldp x21, x22, [sp, #32] 0.83 : ffff8000104b0820: ldp x23, x24, [sp, #48] 0.00 : ffff8000104b0824: ldp x25, x26, [sp, #64] 0.00 : ffff8000104b0828: ldp x29, x30, [sp], #96 0.00 : ffff8000104b082c: ret : __ll_sc_atomic64_fetch_or_acquire(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000104b0830: b ffff8000104b1408 <__sbitmap_queue_get+0x170> 0.00 : ffff8000104b0834: b ffff8000104b07f8 <__sbitmap_get_word+0xd8> : __sbitmap_get_word(): : return -1; 0.00 : ffff8000104b0838: mov w4, #0xffffffff // #-1 0.00 : ffff8000104b083c: b ffff8000104b0800 <__sbitmap_get_word+0xe0> : } 0.00 : ffff8000104b0840: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (100 samples, percent: local period) ---------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d86b8 : : aio_prep_rw(): : iocb->ki_res.res2 = res2; : iocb_put(iocb); : } : : static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) : { 0.00 : ffff8000102d86b8: stp x29, x30, [sp, #-32]! 0.00 : ffff8000102d86bc: mov x29, sp 8.09 : ffff8000102d86c0: stp x19, x20, [sp, #16] 0.00 : ffff8000102d86c4: mov x19, x0 : int ret; : : req->ki_complete = aio_complete_rw; 0.00 : ffff8000102d86c8: adrp x0, ffff8000102da000 0.00 : ffff8000102d86cc: add x0, x0, #0x3a0 : { 3.05 : ffff8000102d86d0: mov x20, x1 : req->private = NULL; 0.00 : ffff8000102d86d4: stp x0, xzr, [x19, #16] : req->ki_pos = iocb->aio_offset; : req->ki_flags = iocb_flags(req->ki_filp); 3.03 : ffff8000102d86d8: ldr x2, [x19] : req->ki_pos = iocb->aio_offset; 0.00 : ffff8000102d86dc: ldr x0, [x1, #40] 0.00 : ffff8000102d86e0: str x0, [x19, #8] : req->ki_flags = iocb_flags(req->ki_filp); 0.00 : ffff8000102d86e4: ldr w1, [x2, #64] : iocb_flags(): : return true; : } : : static inline int iocb_flags(struct file *file) : { : int res = 0; 0.00 : ffff8000102d86e8: ubfx x0, x1, #10, #1 : if (file->f_flags & O_APPEND) : res |= IOCB_APPEND; : if (io_is_direct(file)) : res |= IOCB_DIRECT; 0.00 : ffff8000102d86ec: tst x1, #0x10000 : int res = 0; 0.00 : ffff8000102d86f0: lsl w0, w0, #1 : res |= IOCB_DIRECT; 0.00 : ffff8000102d86f4: orr w3, w0, #0x4 0.00 : ffff8000102d86f8: csel w0, w3, w0, ne // ne = any : if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host)) 0.00 : ffff8000102d86fc: tbnz w1, #12, ffff8000102d8714 0.00 : ffff8000102d8700: ldr x3, [x2, #240] 0.00 : ffff8000102d8704: ldr x3, [x3] 5.88 : ffff8000102d8708: ldr x4, [x3, #40] 1.00 : ffff8000102d870c: ldr x4, [x4, #80] 0.00 : ffff8000102d8710: tbz w4, #4, ffff8000102d8818 : res |= IOCB_DSYNC; 0.00 : ffff8000102d8714: orr w0, w0, #0x10 : if (file->f_flags & __O_SYNC) : res |= IOCB_SYNC; 2.99 : ffff8000102d8718: tst x1, #0x100000 0.00 : ffff8000102d871c: orr w3, w0, #0x20 0.00 : ffff8000102d8720: csel w0, w3, w0, ne // ne = any : aio_prep_rw(): 0.00 : ffff8000102d8724: str w0, [x19, #32] : if (iocb->aio_flags & IOCB_FLAG_RESFD) 1.02 : ffff8000102d8728: ldr w1, [x20, #56] 0.00 : ffff8000102d872c: tbz w1, #0, ffff8000102d8738 : req->ki_flags |= IOCB_EVENTFD; 0.00 : ffff8000102d8730: orr w0, w0, #0x1 0.00 : ffff8000102d8734: str w0, [x19, #32] : file_write_hint(): : if (file->f_write_hint != WRITE_LIFE_NOT_SET) 0.00 : ffff8000102d8738: ldr w0, [x2, #52] 0.00 : ffff8000102d873c: cbnz w0, ffff8000102d87dc : return file_inode(file)->i_write_hint; 2.02 : ffff8000102d8740: ldr x0, [x2, #32] 0.00 : ffff8000102d8744: ldrb w0, [x0, #143] : ki_hint_validate(): : return hint; 0.00 : ffff8000102d8748: and w1, w0, #0xffff : aio_prep_rw(): : req->ki_hint = ki_hint_validate(file_write_hint(req->ki_filp)); 15.88 : ffff8000102d874c: strh w1, [x19, #36] : if (iocb->aio_flags & IOCB_FLAG_IOPRIO) { 0.00 : ffff8000102d8750: ldr w0, [x20, #56] 0.00 : ffff8000102d8754: tbnz w0, #1, ffff8000102d87f8 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 2.04 : ffff8000102d8758: mrs x0, sp_el0 : get_current_ioprio(): : * If the calling process has set an I/O priority, use that. Otherwise, return : * the default I/O priority. : */ : static inline int get_current_ioprio(void) : { : struct io_context *ioc = current->io_context; 0.00 : ffff8000102d875c: ldr x1, [x0, #1864] : : if (ioc) 0.00 : ffff8000102d8760: mov w0, #0x0 // #0 0.00 : ffff8000102d8764: cbz x1, ffff8000102d876c : return ioc->ioprio; 1.01 : ffff8000102d8768: ldrh w0, [x1, #20] : aio_prep_rw(): : return ret; : } : : req->ki_ioprio = iocb->aio_reqprio; : } else : req->ki_ioprio = get_current_ioprio(); 6.84 : ffff8000102d876c: strh w0, [x19, #38] : : ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags); 0.00 : ffff8000102d8770: ldr w2, [x20, #12] : kiocb_set_rw_flags(): : return res; : } : : static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags) : { : if (unlikely(flags & ~RWF_SUPPORTED)) 0.00 : ffff8000102d8774: ands w0, w2, #0xffffffe0 0.00 : ffff8000102d8778: b.ne ffff8000102d8824 // b.any : return -EOPNOTSUPP; : : if (flags & RWF_NOWAIT) { 0.00 : ffff8000102d877c: tbz w2, #3, ffff8000102d8810 : if (!(ki->ki_filp->f_mode & FMODE_NOWAIT)) 0.00 : ffff8000102d8780: ldr x1, [x19] 0.00 : ffff8000102d8784: ldr w1, [x1, #68] 0.00 : ffff8000102d8788: tbz w1, #27, ffff8000102d8824 : return -EOPNOTSUPP; : ki->ki_flags |= IOCB_NOWAIT; 0.00 : ffff8000102d878c: ldr w1, [x19, #32] 0.00 : ffff8000102d8790: orr w1, w1, #0x80 0.00 : ffff8000102d8794: str w1, [x19, #32] : } : if (flags & RWF_HIPRI) 10.06 : ffff8000102d8798: tbz w2, #0, ffff8000102d87a4 : ki->ki_flags |= IOCB_HIPRI; 0.00 : ffff8000102d879c: orr w1, w1, #0x8 0.00 : ffff8000102d87a0: str w1, [x19, #32] : if (flags & RWF_DSYNC) 0.00 : ffff8000102d87a4: tbz w2, #1, ffff8000102d87b0 : ki->ki_flags |= IOCB_DSYNC; 0.00 : ffff8000102d87a8: orr w1, w1, #0x10 0.00 : ffff8000102d87ac: str w1, [x19, #32] : if (flags & RWF_SYNC) 0.00 : ffff8000102d87b0: tbz w2, #2, ffff8000102d87bc : ki->ki_flags |= (IOCB_DSYNC | IOCB_SYNC); 0.00 : ffff8000102d87b4: orr w1, w1, #0x30 0.00 : ffff8000102d87b8: str w1, [x19, #32] : if (flags & RWF_APPEND) : ki->ki_flags |= IOCB_APPEND; 0.00 : ffff8000102d87bc: orr w3, w1, #0x2 0.00 : ffff8000102d87c0: tst x2, #0x10 0.00 : ffff8000102d87c4: csel w1, w3, w1, ne // ne = any : aio_prep_rw(): : if (unlikely(ret)) : return ret; : : req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ 0.00 : ffff8000102d87c8: and w1, w1, #0xfffffff7 0.98 : ffff8000102d87cc: str w1, [x19, #32] : return 0; : } 4.07 : ffff8000102d87d0: ldp x19, x20, [sp, #16] 2.02 : ffff8000102d87d4: ldp x29, x30, [sp], #32 0.00 : ffff8000102d87d8: ret : ki_hint_validate(): : if (hint <= max_hint) 0.00 : ffff8000102d87dc: mov w2, #0xffff // #65535 : return 0; 0.00 : ffff8000102d87e0: mov w1, #0x0 // #0 : if (hint <= max_hint) 0.00 : ffff8000102d87e4: cmp w0, w2 0.00 : ffff8000102d87e8: b.ls ffff8000102d8748 // b.plast : aio_prep_rw(): : req->ki_hint = ki_hint_validate(file_write_hint(req->ki_filp)); 0.00 : ffff8000102d87ec: strh w1, [x19, #36] : if (iocb->aio_flags & IOCB_FLAG_IOPRIO) { 0.00 : ffff8000102d87f0: ldr w0, [x20, #56] 0.00 : ffff8000102d87f4: tbz w0, #1, ffff8000102d8758 : ret = ioprio_check_cap(iocb->aio_reqprio); 0.00 : ffff8000102d87f8: ldrsh w0, [x20, #18] 0.00 : ffff8000102d87fc: bl ffff800010473468 : if (ret) { 0.00 : ffff8000102d8800: cbnz w0, ffff8000102d87d0 : req->ki_ioprio = iocb->aio_reqprio; 0.00 : ffff8000102d8804: ldrh w0, [x20, #18] 0.00 : ffff8000102d8808: strh w0, [x19, #38] 0.00 : ffff8000102d880c: b ffff8000102d8770 7.12 : ffff8000102d8810: ldr w1, [x19, #32] 0.00 : ffff8000102d8814: b ffff8000102d8798 : iocb_flags(): : if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host)) 22.91 : ffff8000102d8818: ldr w3, [x3, #12] 0.00 : ffff8000102d881c: tbz w3, #0, ffff8000102d8718 0.00 : ffff8000102d8820: b ffff8000102d8714 : kiocb_set_rw_flags(): : return -EOPNOTSUPP; 0.00 : ffff8000102d8824: mov w0, #0xffffffa1 // #-95 0.00 : ffff8000102d8828: b ffff8000102d87d0 Percent | Source code & Disassembly of vmlinux for cycles (39 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b260 : : update_ts_time_stats(): : /* : * Updates the per-CPU time idle statistics counters : */ : static void : update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time) : { 0.00 : ffff80001017b260: stp x29, x30, [sp, #-48]! 0.00 : ffff80001017b264: mov x29, sp 0.00 : ffff80001017b268: stp x19, x20, [sp, #16] 0.00 : ffff80001017b26c: mov x20, x1 5.26 : ffff80001017b270: str x21, [sp, #32] 0.00 : ffff80001017b274: mov x19, x2 0.00 : ffff80001017b278: mov x21, x3 : ktime_t delta; : : if (ts->idle_active) { 4.79 : ffff80001017b27c: ldrb w1, [x1, #76] 0.00 : ffff80001017b280: tbnz w1, #2, ffff80001017b2b8 : else : ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); : ts->idle_entrytime = now; : } : : if (last_update_time) 0.00 : ffff80001017b284: cbz x21, ffff80001017b2a8 : ktime_divns(): : /* : * 32-bit implementation cannot handle negative divisors, : * so catch them on 64bit as well. : */ : WARN_ON(div < 0); : return kt / div; 0.00 : ffff80001017b288: mov x2, #0xf7cf // #63439 0.00 : ffff80001017b28c: movk x2, #0xe353, lsl #16 0.00 : ffff80001017b290: movk x2, #0x9ba5, lsl #32 0.00 : ffff80001017b294: movk x2, #0x20c4, lsl #48 0.00 : ffff80001017b298: smulh x2, x19, x2 0.00 : ffff80001017b29c: asr x2, x2, #7 0.00 : ffff80001017b2a0: sub x19, x2, x19, asr #63 : update_ts_time_stats(): : *last_update_time = ktime_to_us(now); 0.00 : ffff80001017b2a4: str x19, [x21] : : } 7.61 : ffff80001017b2a8: ldp x19, x20, [sp, #16] 0.00 : ffff80001017b2ac: ldr x21, [sp, #32] 0.00 : ffff80001017b2b0: ldp x29, x30, [sp], #48 0.00 : ffff80001017b2b4: ret 0.00 : ffff80001017b2b8: str x22, [x29, #40] : delta = ktime_sub(now, ts->idle_entrytime); 2.43 : ffff80001017b2bc: ldr x22, [x20, #120] 0.00 : ffff80001017b2c0: sub x22, x2, x22 : if (nr_iowait_cpu(cpu) > 0) 0.00 : ffff80001017b2c4: bl ffff800010117f78 46.16 : ffff80001017b2c8: cbnz x0, ffff80001017b2e4 : ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); 0.00 : ffff80001017b2cc: ldr x0, [x20, #144] 0.00 : ffff80001017b2d0: add x22, x0, x22 33.76 : ffff80001017b2d4: str x22, [x20, #144] : ts->idle_entrytime = now; 0.00 : ffff80001017b2d8: str x19, [x20, #120] 0.00 : ffff80001017b2dc: ldr x22, [x29, #40] 0.00 : ffff80001017b2e0: b ffff80001017b284 : ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); 0.00 : ffff80001017b2e4: ldr x0, [x20, #152] 0.00 : ffff80001017b2e8: add x22, x0, x22 0.00 : ffff80001017b2ec: str x22, [x20, #152] 0.00 : ffff80001017b2f0: b ffff80001017b2d8 Percent | Source code & Disassembly of vmlinux for cycles (36 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101c2450 : : perf_pmu_disable(): : : return 0; : } : : void perf_pmu_disable(struct pmu *pmu) : { 22.43 : ffff8000101c2450: stp x29, x30, [sp, #-16]! 0.00 : ffff8000101c2454: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000101c2458: mrs x4, tpidr_el1 : perf_pmu_disable(): : int *count = this_cpu_ptr(pmu->pmu_disable_count); 0.00 : ffff8000101c245c: ldr x1, [x0, #64] : if (!(*count)++) 41.56 : ffff8000101c2460: ldr w3, [x1, x4] 0.00 : ffff8000101c2464: add w5, w3, #0x1 36.01 : ffff8000101c2468: str w5, [x1, x4] 0.00 : ffff8000101c246c: cbnz w3, ffff8000101c2478 : pmu->pmu_disable(pmu); 0.00 : ffff8000101c2470: ldr x1, [x0, #104] 0.00 : ffff8000101c2474: blr x1 : } 0.00 : ffff8000101c2478: ldp x29, x30, [sp], #16 0.00 : ffff8000101c247c: ret Percent | Source code & Disassembly of vmlinux for cycles (94 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101dc538 : : set_page_dirty_lock(): : * holds a reference on the inode by having an open file. : * : * In other cases, the page should be locked before running set_page_dirty(). : */ : int set_page_dirty_lock(struct page *page) : { 0.00 : ffff8000101dc538: stp x29, x30, [sp, #-64]! 0.00 : ffff8000101dc53c: mov x29, sp 1.06 : ffff8000101dc540: stp x19, x20, [sp, #16] 0.00 : ffff8000101dc544: adrp x20, ffff800011909000 16.05 : ffff8000101dc548: str x21, [sp, #32] 0.00 : ffff8000101dc54c: add x1, x20, #0x908 0.00 : ffff8000101dc550: mov x19, x0 2.14 : ffff8000101dc554: ldr x0, [x1] 0.00 : ffff8000101dc558: str x0, [x29, #56] 0.00 : ffff8000101dc55c: mov x0, #0x0 // #0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.13 : ffff8000101dc560: ldr x1, [x19, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff8000101dc564: sub x0, x1, #0x1 0.00 : ffff8000101dc568: tst x1, #0x1 0.00 : ffff8000101dc56c: csel x0, x0, x19, ne // ne = any : __read_once_size(): 0.00 : ffff8000101dc570: ldr x1, [x0] : test_and_set_bit_lock(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (READ_ONCE(*p) & mask) 0.00 : ffff8000101dc574: tbz w1, #0, ffff8000101dc5bc : lock_page(): : */ : static inline void lock_page(struct page *page) : { : might_sleep(); : if (!trylock_page(page)) : __lock_page(page); 0.00 : ffff8000101dc578: mov x0, x19 0.00 : ffff8000101dc57c: bl ffff8000101d18b0 <__lock_page> : set_page_dirty_lock(): : int ret; : : lock_page(page); : ret = set_page_dirty(page); 2.14 : ffff8000101dc580: mov x0, x19 0.00 : ffff8000101dc584: bl ffff8000101dc410 : unlock_page(page); : return ret; : } 0.00 : ffff8000101dc588: add x20, x20, #0x908 : ret = set_page_dirty(page); 0.00 : ffff8000101dc58c: mov w21, w0 : unlock_page(page); 0.00 : ffff8000101dc590: mov x0, x19 0.00 : ffff8000101dc594: bl ffff8000101d1588 : } 1.10 : ffff8000101dc598: mov w0, w21 0.00 : ffff8000101dc59c: ldr x2, [x29, #56] 0.00 : ffff8000101dc5a0: ldr x1, [x20] 0.00 : ffff8000101dc5a4: eor x1, x2, x1 0.00 : ffff8000101dc5a8: cbnz x1, ffff8000101dc5e8 2.14 : ffff8000101dc5ac: ldp x19, x20, [sp, #16] 0.00 : ffff8000101dc5b0: ldr x21, [sp, #32] 0.00 : ffff8000101dc5b4: ldp x29, x30, [sp], #64 0.00 : ffff8000101dc5b8: ret : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 17.09 : ffff8000101dc5bc: b ffff8000101dc5d8 9.59 : ffff8000101dc5c0: b ffff8000101dc5d8 : __lse_atomic64_fetch_or_acquire(): : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff8000101dc5c4: mov x1, #0x1 // #1 0.00 : ffff8000101dc5c8: ldseta x1, x1, [x0] 46.56 : ffff8000101dc5cc: mov x0, x1 : lock_page(): : if (!trylock_page(page)) 0.00 : ffff8000101dc5d0: tbz w0, #0, ffff8000101dc580 0.00 : ffff8000101dc5d4: b ffff8000101dc578 : __ll_sc_atomic64_fetch_or_acquire(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000101dc5d8: b ffff8000101deae4 0.00 : ffff8000101dc5dc: mov x0, x1 : lock_page(): 0.00 : ffff8000101dc5e0: tbz w0, #0, ffff8000101dc580 0.00 : ffff8000101dc5e4: b ffff8000101dc578 : set_page_dirty_lock(): 0.00 : ffff8000101dc5e8: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (38 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011bde0 : : pick_next_task_idle(): : struct task_struct *next = rq->idle; : : set_next_task_idle(rq, next, true); : : return next; : } 100.00 : ffff80001011bde0: ldr x0, [x0, #2360] 0.00 : ffff80001011bde4: ret Percent | Source code & Disassembly of vmlinux for cycles (37 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011c3f8 : : cpu_startup_entry(): : preempt_enable(); : } : EXPORT_SYMBOL_GPL(play_idle_precise); : : void cpu_startup_entry(enum cpuhp_state state) : { 0.00 : ffff80001011c3f8: stp x29, x30, [sp, #-32]! 0.00 : ffff80001011c3fc: mov x29, sp 0.00 : ffff80001011c400: str x19, [sp, #16] 0.00 : ffff80001011c404: mov w19, w0 : arch_cpu_idle_prepare(); 0.00 : ffff80001011c408: bl ffff80001011bf28 : cpuhp_online_idle(state); 0.00 : ffff80001011c40c: mov w0, w19 0.00 : ffff80001011c410: bl ffff8000100e8b28 : while (1) : do_idle(); 11.11 : ffff80001011c414: bl ffff80001011bf70 47.99 : ffff80001011c418: bl ffff80001011bf70 40.89 : ffff80001011c41c: b ffff80001011c414 Percent | Source code & Disassembly of vmlinux for cycles (80 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010467df0 : : blk_mq_flush_plug_list(): : : return blk_rq_pos(rqa) > blk_rq_pos(rqb); : } : : void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) : { 6.01 : ffff800010467df0: stp x29, x30, [sp, #-112]! 0.00 : ffff800010467df4: mov x29, sp 11.40 : ffff800010467df8: stp x19, x20, [sp, #16] 0.00 : ffff800010467dfc: mov x20, x0 1.26 : ffff800010467e00: str x21, [sp, #32] 0.00 : ffff800010467e04: adrp x21, ffff800011909000 0.00 : ffff800010467e08: add x2, x21, #0x908 : LIST_HEAD(list); 0.00 : ffff800010467e0c: add x19, x29, #0x48 : { 10.15 : ffff800010467e10: ldr x0, [x2] 5.04 : ffff800010467e14: str x0, [x29, #104] 0.00 : ffff800010467e18: mov x0, #0x0 // #0 : LIST_HEAD(list); 0.00 : ffff800010467e1c: stp x19, x19, [x29, #72] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.30 : ffff800010467e20: ldr x0, [x20] : blk_mq_flush_plug_list(): : : if (list_empty(&plug->mq_list)) 0.00 : ffff800010467e24: cmp x20, x0 0.00 : ffff800010467e28: b.eq ffff800010467f18 // b.none 0.00 : ffff800010467e2c: stp x22, x23, [x29, #40] 0.00 : ffff800010467e30: and w23, w1, #0xff : __read_once_size(): 2.45 : ffff800010467e34: ldr x0, [x20] : list_splice_init(): : * The list at @list is reinitialised : */ : static inline void list_splice_init(struct list_head *list, : struct list_head *head) : { : if (!list_empty(list)) { 0.00 : ffff800010467e38: cmp x20, x0 0.00 : ffff800010467e3c: b.eq ffff800010467e5c // b.none : __list_splice(): : struct list_head *last = list->prev; 3.81 : ffff800010467e40: ldp x1, x0, [x20] : first->prev = prev; 2.51 : ffff800010467e44: str x19, [x1, #8] : prev->next = first; 10.05 : ffff800010467e48: str x1, [x29, #72] : last->next = next; 0.00 : ffff800010467e4c: str x19, [x0] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 8.81 : ffff800010467e50: str x20, [x20] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010467e54: str x20, [x20, #8] : __list_splice(): : next->prev = last; 0.00 : ffff800010467e58: str x0, [x29, #80] : blk_mq_flush_plug_list(): : return; : list_splice_init(&plug->mq_list, &list); : : if (plug->rq_count > 2 && plug->multiple_queues) 0.00 : ffff800010467e5c: ldrh w0, [x20, #32] 0.00 : ffff800010467e60: cmp w0, #0x2 0.00 : ffff800010467e64: b.hi ffff800010467f3c // b.pmore : list_cut_before(): : list->next->prev = list; 1.08 : ffff800010467e68: add x22, x29, #0x58 : blk_mq_flush_plug_list(): : list_sort(NULL, &list, plug_rq_cmp); : : plug->rq_count = 0; 0.00 : ffff800010467e6c: strh wzr, [x20, #32] : : do { : struct list_head rq_list; : struct request *rq, *head_rq = list_entry_rq(list.next); 0.00 : ffff800010467e70: ldr x5, [x29, #72] : struct list_head *pos = &head_rq->queuelist; /* skip first */ : struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx; : struct blk_mq_ctx *this_ctx = head_rq->mq_ctx; : unsigned int depth = 1; : : list_for_each_continue(pos, &list) { 5.06 : ffff800010467e74: ldr x2, [x5] : struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx; 7.57 : ffff800010467e78: ldp x1, x0, [x5, #-64] : list_for_each_continue(pos, &list) { 0.00 : ffff800010467e7c: cmp x2, x19 6.32 : ffff800010467e80: b.eq ffff800010467edc // b.none : rq = list_entry_rq(pos); : BUG_ON(!rq->q); 0.00 : ffff800010467e84: ldur x3, [x2, #-72] 0.00 : ffff800010467e88: cbz x3, ffff800010467ed8 : if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx) 0.00 : ffff800010467e8c: ldur x3, [x2, #-56] 0.00 : ffff800010467e90: cmp x0, x3 0.00 : ffff800010467e94: b.ne ffff800010467edc // b.any 0.00 : ffff800010467e98: ldur x3, [x2, #-64] 0.00 : ffff800010467e9c: cmp x1, x3 0.00 : ffff800010467ea0: b.eq ffff800010467ec0 // b.none 0.00 : ffff800010467ea4: b ffff800010467edc 0.00 : ffff800010467ea8: ldur x3, [x2, #-56] 0.00 : ffff800010467eac: cmp x3, x0 0.00 : ffff800010467eb0: b.ne ffff800010467edc // b.any 0.00 : ffff800010467eb4: ldr x3, [x4, #8] 0.00 : ffff800010467eb8: cmp x3, x1 0.00 : ffff800010467ebc: b.ne ffff800010467edc // b.any : list_for_each_continue(pos, &list) { 0.00 : ffff800010467ec0: ldr x2, [x2] 0.00 : ffff800010467ec4: cmp x2, x19 0.00 : ffff800010467ec8: b.eq ffff800010467edc // b.none : BUG_ON(!rq->q); 0.00 : ffff800010467ecc: ldur x3, [x2, #-72] 0.00 : ffff800010467ed0: sub x4, x2, #0x48 0.00 : ffff800010467ed4: cbnz x3, ffff800010467ea8 0.00 : ffff800010467ed8: brk #0x800 : list_cut_before(): : if (head->next == entry) { 8.74 : ffff800010467edc: cmp x2, x5 0.00 : ffff800010467ee0: b.eq ffff800010467f5c // b.none : list->next->prev = list; 0.00 : ffff800010467ee4: str x22, [x5, #8] : list->prev = entry->prev; 0.00 : ffff800010467ee8: ldr x3, [x2, #8] 0.00 : ffff800010467eec: stp x5, x3, [x29, #88] : list->prev->next = list; 0.00 : ffff800010467ef0: str x22, [x3] : entry->prev = head; 0.00 : ffff800010467ef4: str x19, [x2, #8] : head->next = entry; 0.00 : ffff800010467ef8: str x2, [x29, #72] : blk_mq_flush_plug_list(): : depth++; : } : : list_cut_before(&rq_list, &list, pos); : trace_block_unplug(head_rq->q, depth, !from_schedule); : blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list, 0.00 : ffff800010467efc: mov w3, w23 0.00 : ffff800010467f00: mov x2, x22 0.00 : ffff800010467f04: bl ffff80001046ce80 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010467f08: ldr x0, [x29, #72] : blk_mq_flush_plug_list(): : from_schedule); : } while(!list_empty(&list)); 0.00 : ffff800010467f0c: cmp x19, x0 0.00 : ffff800010467f10: b.ne ffff800010467e70 // b.any 0.00 : ffff800010467f14: ldp x22, x23, [x29, #40] : } 0.00 : ffff800010467f18: add x21, x21, #0x908 3.69 : ffff800010467f1c: ldr x1, [x29, #104] 0.00 : ffff800010467f20: ldr x0, [x21] 0.00 : ffff800010467f24: eor x0, x1, x0 0.00 : ffff800010467f28: cbnz x0, ffff800010467f68 3.59 : ffff800010467f2c: ldp x19, x20, [sp, #16] 0.00 : ffff800010467f30: ldr x21, [sp, #32] 1.15 : ffff800010467f34: ldp x29, x30, [sp], #112 0.00 : ffff800010467f38: ret : if (plug->rq_count > 2 && plug->multiple_queues) 0.00 : ffff800010467f3c: ldrb w0, [x20, #34] 0.00 : ffff800010467f40: cbz w0, ffff800010467e68 : list_sort(NULL, &list, plug_rq_cmp); 0.00 : ffff800010467f44: adrp x2, ffff800010464000 0.00 : ffff800010467f48: mov x1, x19 0.00 : ffff800010467f4c: add x2, x2, #0x780 0.00 : ffff800010467f50: mov x0, #0x0 // #0 0.00 : ffff800010467f54: bl ffff800010487910 0.00 : ffff800010467f58: b ffff800010467e68 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010467f5c: str x22, [x29, #88] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010467f60: str x22, [x29, #96] 0.00 : ffff800010467f64: b ffff800010467efc 0.00 : ffff800010467f68: stp x22, x23, [x29, #40] : blk_mq_flush_plug_list(): : } 0.00 : ffff800010467f6c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (33 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b488 : : tick_do_update_jiffies64(): : static ktime_t last_jiffies_update; : : /* : * Must be called with interrupts disabled ! : */ : static void tick_do_update_jiffies64(ktime_t now) 0.00 : ffff80001017b488: stp x29, x30, [sp, #-64]! 0.00 : ffff80001017b48c: mov x29, sp 3.07 : ffff80001017b490: stp x19, x20, [sp, #16] : raw_write_seqcount_begin(): : : : : static inline void raw_write_seqcount_begin(seqcount_t *s) : { : s->sequence++; 0.00 : ffff80001017b494: adrp x20, ffff800011907000 : tick_do_update_jiffies64(): 0.00 : ffff80001017b498: stp x21, x22, [sp, #32] : spin_lock(): : raw_spin_lock_init(&(_lock)->rlock); \ : } while (0) : : static __always_inline void spin_lock(spinlock_t *lock) : { : raw_spin_lock(&lock->rlock); 0.00 : ffff80001017b49c: adrp x21, ffff800011907000 : tick_do_update_jiffies64(): 0.00 : ffff80001017b4a0: stp x23, x24, [sp, #48] : spin_lock(): 0.00 : ffff80001017b4a4: add x24, x21, #0x9c4 : tick_do_update_jiffies64(): 0.00 : ffff80001017b4a8: mov x19, x0 : spin_lock(): 0.00 : ffff80001017b4ac: mov x0, x24 0.00 : ffff80001017b4b0: bl ffff800010cd80f0 <_raw_spin_lock> : raw_write_seqcount_begin(): 0.00 : ffff80001017b4b4: ldr w0, [x20, #2496] 0.00 : ffff80001017b4b8: add w0, w0, #0x1 9.01 : ffff80001017b4bc: str w0, [x20, #2496] : smp_wmb(); 0.00 : ffff80001017b4c0: dmb ishst : tick_do_update_jiffies64(): : return; : : /* Reevaluate with jiffies_lock held */ : write_seqlock(&jiffies_lock); : : delta = ktime_sub(now, last_jiffies_update); 21.13 : ffff80001017b4c4: adrp x22, ffff800011b22000 : if (delta >= tick_period) { 0.00 : ffff80001017b4c8: adrp x23, ffff800011b22000 : delta = ktime_sub(now, last_jiffies_update); 2.97 : ffff80001017b4cc: ldr x1, [x22, #1328] : if (delta >= tick_period) { 2.96 : ffff80001017b4d0: ldr x2, [x23, #968] : delta = ktime_sub(now, last_jiffies_update); 0.00 : ffff80001017b4d4: sub x0, x19, x1 : if (delta >= tick_period) { 0.00 : ffff80001017b4d8: cmp x0, x2 0.00 : ffff80001017b4dc: b.lt ffff80001017b540 // b.tstop : : delta = ktime_sub(delta, tick_period); : /* Pairs with the lockless read in this function. */ : WRITE_ONCE(last_jiffies_update, 0.00 : ffff80001017b4e0: add x3, x1, x2 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001017b4e4: str x3, [x22, #1328] : tick_do_update_jiffies64(): : delta = ktime_sub(delta, tick_period); 0.00 : ffff80001017b4e8: sub x1, x0, x2 : ktime_add(last_jiffies_update, tick_period)); : : /* Slow path for long timeouts */ : if (unlikely(delta >= tick_period)) { 0.00 : ffff80001017b4ec: mov x0, #0x1 // #1 0.00 : ffff80001017b4f0: cmp x2, x1 0.00 : ffff80001017b4f4: b.le ffff80001017b56c : /* Pairs with the lockless read in this function. */ : WRITE_ONCE(last_jiffies_update, : ktime_add_ns(last_jiffies_update, : incr * ticks)); : } : do_timer(++ticks); 0.00 : ffff80001017b4f8: bl ffff80001016ea08 : : /* Keep the tick_next_period variable up to date */ : tick_next_period = ktime_add(last_jiffies_update, tick_period); 0.00 : ffff80001017b4fc: adrp x1, ffff800011b22000 0.00 : ffff80001017b500: ldr x0, [x22, #1328] 0.00 : ffff80001017b504: ldr x2, [x23, #968] 0.00 : ffff80001017b508: add x0, x0, x2 0.00 : ffff80001017b50c: str x0, [x1, #960] : raw_write_seqcount_end(): : } : : static inline void raw_write_seqcount_end(seqcount_t *s) : { : smp_wmb(); 0.00 : ffff80001017b510: dmb ishst : s->sequence++; 60.86 : ffff80001017b514: ldr w1, [x20, #2496] : spin_unlock(): : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ : } while (0) : : static __always_inline void spin_unlock(spinlock_t *lock) : { : raw_spin_unlock(&lock->rlock); 0.00 : ffff80001017b518: add x0, x21, #0x9c4 : raw_write_seqcount_end(): 0.00 : ffff80001017b51c: add w1, w1, #0x1 0.00 : ffff80001017b520: str w1, [x20, #2496] : spin_unlock(): 0.00 : ffff80001017b524: bl ffff800010cd7c38 <_raw_spin_unlock> : tick_do_update_jiffies64(): : } else { : write_sequnlock(&jiffies_lock); : return; : } : write_sequnlock(&jiffies_lock); : update_wall_time(); 0.00 : ffff80001017b528: bl ffff80001016e9f0 : } 0.00 : ffff80001017b52c: ldp x19, x20, [sp, #16] 0.00 : ffff80001017b530: ldp x21, x22, [sp, #32] 0.00 : ffff80001017b534: ldp x23, x24, [sp, #48] 0.00 : ffff80001017b538: ldp x29, x30, [sp], #64 0.00 : ffff80001017b53c: ret : raw_write_seqcount_end(): : smp_wmb(); 0.00 : ffff80001017b540: dmb ishst : s->sequence++; 0.00 : ffff80001017b544: ldr w1, [x20, #2496] : spin_unlock(): 0.00 : ffff80001017b548: mov x0, x24 : raw_write_seqcount_end(): 0.00 : ffff80001017b54c: add w1, w1, #0x1 0.00 : ffff80001017b550: str w1, [x20, #2496] : spin_unlock(): 0.00 : ffff80001017b554: bl ffff800010cd7c38 <_raw_spin_unlock> : tick_do_update_jiffies64(): 0.00 : ffff80001017b558: ldp x19, x20, [sp, #16] 0.00 : ffff80001017b55c: ldp x21, x22, [sp, #32] 0.00 : ffff80001017b560: ldp x23, x24, [sp, #48] 0.00 : ffff80001017b564: ldp x29, x30, [sp], #64 0.00 : ffff80001017b568: ret : ktime_divns(): : { : /* : * 32-bit implementation cannot handle negative divisors, : * so catch them on 64bit as well. : */ : WARN_ON(div < 0); 0.00 : ffff80001017b56c: tbnz x2, #63, ffff80001017b584 : return kt / div; 0.00 : ffff80001017b570: sdiv x0, x1, x2 : tick_do_update_jiffies64(): : WRITE_ONCE(last_jiffies_update, 0.00 : ffff80001017b574: madd x2, x2, x0, x3 0.00 : ffff80001017b578: add x0, x0, #0x1 : __write_once_size(): 0.00 : ffff80001017b57c: str x2, [x22, #1328] 0.00 : ffff80001017b580: b ffff80001017b4f8 : ktime_divns(): : WARN_ON(div < 0); 0.00 : ffff80001017b584: brk #0x800 0.00 : ffff80001017b588: ldr x3, [x22, #1328] 0.00 : ffff80001017b58c: b ffff80001017b570 Percent | Source code & Disassembly of vmlinux for cycles (60 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010160210 : : rcu_preempt_deferred_qs(): : * not being in an RCU read-side critical section. The caller must : * evaluate safety in terms of interrupt, softirq, and preemption : * disabling. : */ : static void rcu_preempt_deferred_qs(struct task_struct *t) : { 21.64 : ffff800010160210: stp x29, x30, [sp, #-32]! 0.00 : ffff800010160214: mov x4, x0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 22.18 : ffff800010160218: mrs x3, sp_el0 : rcu_preempt_deferred_qs(): 0.00 : ffff80001016021c: mov x29, sp 6.40 : ffff800010160220: str x19, [sp, #16] : unsigned long flags; : bool couldrecurse = rcu_preempt_depth() >= 0; 27.51 : ffff800010160224: ldr w19, [x3, #776] : : if (!rcu_preempt_need_deferred_qs(t)) 0.00 : ffff800010160228: bl ffff80001015df30 5.01 : ffff80001016022c: tst w0, #0xff 0.00 : ffff800010160230: b.eq ffff800010160274 // b.none : return; : if (couldrecurse) 0.00 : ffff800010160234: tbnz w19, #31, ffff800010160248 : rcu_preempt_depth_set(rcu_preempt_depth() - RCU_NEST_BIAS); 0.00 : ffff800010160238: ldr w0, [x3, #776] 0.00 : ffff80001016023c: mov w1, #0x80000001 // #-2147483647 0.00 : ffff800010160240: add w0, w0, w1 : rcu_preempt_depth_set(): : current->rcu_read_lock_nesting = val; 0.00 : ffff800010160244: str w0, [x3, #776] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010160248: mrs x1, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001016024c: and w0, w1, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff800010160250: cbz w0, ffff800010160280 : rcu_preempt_deferred_qs(): : local_irq_save(flags); : rcu_preempt_deferred_qs_irqrestore(t, flags); 0.00 : ffff800010160254: mov x0, x4 0.00 : ffff800010160258: bl ffff80001015ff80 : if (couldrecurse) 0.00 : ffff80001016025c: tbnz w19, #31, ffff800010160274 : get_current(): 0.00 : ffff800010160260: mrs x1, sp_el0 : rcu_preempt_deferred_qs(): : rcu_preempt_depth_set(rcu_preempt_depth() + RCU_NEST_BIAS); 0.00 : ffff800010160264: ldr w0, [x1, #776] 0.00 : ffff800010160268: mov w2, #0x7fffffff // #2147483647 0.00 : ffff80001016026c: add w0, w0, w2 : rcu_preempt_depth_set(): : current->rcu_read_lock_nesting = val; 0.00 : ffff800010160270: str w0, [x1, #776] : rcu_preempt_deferred_qs(): : } 1.81 : ffff800010160274: ldr x19, [sp, #16] 15.45 : ffff800010160278: ldp x29, x30, [sp], #32 0.00 : ffff80001016027c: ret : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160280: mov x0, #0x60 // #96 0.00 : ffff800010160284: msr daifset, #0x2 : rcu_preempt_deferred_qs(): : rcu_preempt_deferred_qs_irqrestore(t, flags); 0.00 : ffff800010160288: mov x0, x4 0.00 : ffff80001016028c: bl ffff80001015ff80 : if (couldrecurse) 0.00 : ffff800010160290: tbz w19, #31, ffff800010160260 0.00 : ffff800010160294: b ffff800010160274 Percent | Source code & Disassembly of vmlinux for cycles (70 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd4b28 : : mutex_lock(): : * deadlock debugging) : * : * This function is similar to (but not equivalent to) down(). : */ : void __sched mutex_lock(struct mutex *lock) : { 0.00 : ffff800010cd4b28: stp x29, x30, [sp, #-16]! 0.00 : ffff800010cd4b2c: mov x3, x0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010cd4b30: mrs x2, sp_el0 : mutex_lock(): 0.00 : ffff800010cd4b34: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010cd4b38: b ffff800010cd4b5c 0.00 : ffff800010cd4b3c: b ffff800010cd4b5c : __lse__cmpxchg_case_acq_64(): : __CMPXCHG_CASE(w, , , 32, ) : __CMPXCHG_CASE(x, , , 64, ) : __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") : __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") : __CMPXCHG_CASE(x, , acq_, 64, a, "memory") 0.00 : ffff800010cd4b40: mov x1, #0x0 // #0 0.00 : ffff800010cd4b44: mov x4, x1 2.81 : ffff800010cd4b48: casa x4, x2, [x0] 70.93 : ffff800010cd4b4c: mov x0, x4 : __mutex_trylock_fast(): : if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) 0.00 : ffff800010cd4b50: cbnz x0, ffff800010cd4b68 : mutex_lock(): : might_sleep(); : : if (!__mutex_trylock_fast(lock)) : __mutex_lock_slowpath(lock); : } 26.25 : ffff800010cd4b54: ldp x29, x30, [sp], #16 0.00 : ffff800010cd4b58: ret : __ll_sc__cmpxchg_case_acq_64(): : __CMPXCHG_CASE(w, , , 32, , , , , K) : __CMPXCHG_CASE( , , , 64, , , , , L) : __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) : __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) : __CMPXCHG_CASE( , , acq_, 64, , a, , "memory", L) 0.00 : ffff800010cd4b5c: mov x1, #0x0 // #0 0.00 : ffff800010cd4b60: b ffff800010cd57f0 : __mutex_trylock_fast(): : if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) 0.00 : ffff800010cd4b64: cbz x0, ffff800010cd4b54 0.00 : ffff800010cd4b68: mov x0, x3 : mutex_lock(): : __mutex_lock_slowpath(lock); 0.00 : ffff800010cd4b6c: bl ffff800010cd4b10 <__mutex_lock_slowpath> : } 0.00 : ffff800010cd4b70: ldp x29, x30, [sp], #16 0.00 : ffff800010cd4b74: ret Percent | Source code & Disassembly of vmlinux for cycles (32 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011ddd0 : : pick_next_entity(): : * 3) pick the "last" process, for cache locality : * 4) do not run the "skip" process, if something else is available : */ : static struct sched_entity * : pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) : { 0.00 : ffff80001011ddd0: stp x29, x30, [sp, #-48]! 0.00 : ffff80001011ddd4: mov x29, sp 19.01 : ffff80001011ddd8: stp x19, x20, [sp, #16] 0.00 : ffff80001011dddc: mov x20, x0 0.00 : ffff80001011dde0: str x21, [sp, #32] 0.00 : ffff80001011dde4: mov x19, x1 : __pick_first_entity(): : struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 3.11 : ffff80001011dde8: ldr x0, [x0, #64] 43.71 : ffff80001011ddec: ldr x21, [x20, #96] : if (!left) 0.00 : ffff80001011ddf0: cbz x0, ffff80001011de6c : pick_next_entity(): : : /* : * If curr is set we have to see if its left of the leftmost entity : * still in the tree, provided there was anything in the tree at all. : */ : if (!left || (curr && entity_before(curr, left))) 9.44 : ffff80001011ddf4: subs x2, x0, #0x18 0.00 : ffff80001011ddf8: b.eq ffff80001011de64 // b.none 0.00 : ffff80001011ddfc: cbz x1, ffff80001011df14 : entity_before(): : return (s64)(a->vruntime - b->vruntime) < 0; 0.00 : ffff80001011de00: ldr x3, [x0, #64] 0.00 : ffff80001011de04: ldr x1, [x1, #88] 0.00 : ffff80001011de08: sub x1, x1, x3 : pick_next_entity(): : if (!left || (curr && entity_before(curr, left))) 0.00 : ffff80001011de0c: tbnz x1, #63, ffff80001011de64 : : /* : * Avoid running the skip buddy, if running something else can : * be done without getting too unfair. : */ : if (cfs_rq->skip == se) { 0.00 : ffff80001011de10: cmp x2, x21 0.00 : ffff80001011de14: b.eq ffff80001011deb8 // b.none : } : : /* : * Prefer last buddy, try to return the CPU to a preempted task. : */ : if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) 0.00 : ffff80001011de18: ldr x0, [x20, #88] : if (cfs_rq->skip == se) { 0.00 : ffff80001011de1c: mov x21, x2 2.80 : ffff80001011de20: mov x19, x2 : if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) 0.00 : ffff80001011de24: cbnz x0, ffff80001011de78 : se = cfs_rq->last; : : /* : * Someone really wants this to run. If it's not unfair, run it. : */ : if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) 3.27 : ffff80001011de28: ldr x0, [x20, #80] 0.00 : ffff80001011de2c: cbz x0, ffff80001011de44 0.00 : ffff80001011de30: ldr x0, [x0, #88] 0.00 : ffff80001011de34: mov x1, x21 0.00 : ffff80001011de38: bl ffff80001011dd70 0.00 : ffff80001011de3c: cmp w0, #0x0 0.00 : ffff80001011de40: b.le ffff80001011de94 : se = cfs_rq->next; : : clear_buddies(cfs_rq, se); 6.00 : ffff80001011de44: mov x1, x19 0.00 : ffff80001011de48: mov x0, x20 0.00 : ffff80001011de4c: bl ffff80001011cfb0 : : return se; : } 12.66 : ffff80001011de50: ldr x21, [sp, #32] 0.00 : ffff80001011de54: mov x0, x19 0.00 : ffff80001011de58: ldp x19, x20, [sp, #16] 0.00 : ffff80001011de5c: ldp x29, x30, [sp], #48 0.00 : ffff80001011de60: ret : if (cfs_rq->skip == se) { 0.00 : ffff80001011de64: cmp x19, x21 0.00 : ffff80001011de68: b.eq ffff80001011df04 // b.none 0.00 : ffff80001011de6c: mov x21, x19 : if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) 0.00 : ffff80001011de70: ldr x0, [x20, #88] 0.00 : ffff80001011de74: cbz x0, ffff80001011de28 0.00 : ffff80001011de78: ldr x0, [x0, #88] 0.00 : ffff80001011de7c: mov x1, x21 0.00 : ffff80001011de80: bl ffff80001011dd70 0.00 : ffff80001011de84: cmp w0, #0x0 0.00 : ffff80001011de88: b.gt ffff80001011de28 : se = cfs_rq->last; 0.00 : ffff80001011de8c: ldr x19, [x20, #88] 0.00 : ffff80001011de90: b ffff80001011de28 : se = cfs_rq->next; 0.00 : ffff80001011de94: ldr x19, [x20, #80] : clear_buddies(cfs_rq, se); 0.00 : ffff80001011de98: mov x0, x20 0.00 : ffff80001011de9c: mov x1, x19 0.00 : ffff80001011dea0: bl ffff80001011cfb0 : } 0.00 : ffff80001011dea4: mov x0, x19 0.00 : ffff80001011dea8: ldr x21, [sp, #32] 0.00 : ffff80001011deac: ldp x19, x20, [sp, #16] 0.00 : ffff80001011deb0: ldp x29, x30, [sp], #48 0.00 : ffff80001011deb4: ret : if (se == curr) { 0.00 : ffff80001011deb8: cmp x19, x21 0.00 : ffff80001011debc: b.eq ffff80001011df20 // b.none : __pick_next_entity(): : struct rb_node *next = rb_next(&se->run_node); 0.00 : ffff80001011dec0: bl ffff800010cc5050 : if (!next) 0.00 : ffff80001011dec4: cbz x0, ffff80001011df08 : pick_next_entity(): : if (!second || (curr && entity_before(curr, second))) 0.00 : ffff80001011dec8: subs x1, x0, #0x18 0.00 : ffff80001011decc: b.eq ffff80001011df08 // b.none 0.00 : ffff80001011ded0: cbz x19, ffff80001011df28 0.00 : ffff80001011ded4: ldr x2, [x19, #88] 0.00 : ffff80001011ded8: ldr x0, [x0, #64] 0.00 : ffff80001011dedc: subs x3, x2, x0 0.00 : ffff80001011dee0: csel x2, x2, x0, mi // mi = first 0.00 : ffff80001011dee4: cmp x3, #0x0 0.00 : ffff80001011dee8: csel x19, x19, x1, lt // lt = tstop : if (second && wakeup_preempt_entity(second, left) < 1) 0.00 : ffff80001011deec: mov x1, x21 0.00 : ffff80001011def0: mov x0, x2 0.00 : ffff80001011def4: bl ffff80001011dd70 0.00 : ffff80001011def8: cmp w0, #0x0 0.00 : ffff80001011defc: csel x19, x19, x21, le 0.00 : ffff80001011df00: b ffff80001011de70 : __pick_first_entity(): : return rb_entry(left, struct sched_entity, run_node); 0.00 : ffff80001011df04: mov x19, x2 : pick_next_entity(): : if (second && wakeup_preempt_entity(second, left) < 1) 0.00 : ffff80001011df08: cbnz x19, ffff80001011df20 0.00 : ffff80001011df0c: mov x19, x21 0.00 : ffff80001011df10: b ffff80001011de70 : if (cfs_rq->skip == se) { 0.00 : ffff80001011df14: cmp x2, x21 0.00 : ffff80001011df18: b.ne ffff80001011de18 // b.any 0.00 : ffff80001011df1c: b ffff80001011dec0 0.00 : ffff80001011df20: ldr x2, [x19, #88] 0.00 : ffff80001011df24: b ffff80001011deec : if (!second || (curr && entity_before(curr, second))) 0.00 : ffff80001011df28: mov x19, x1 0.00 : ffff80001011df2c: ldr x2, [x0, #64] 0.00 : ffff80001011df30: b ffff80001011deec Percent | Source code & Disassembly of vmlinux for cycles (49 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100853a8 <__put_cpu_fpsimd_context>: : __put_cpu_fpsimd_context(): : __get_cpu_fpsimd_context(); : } : : static void __put_cpu_fpsimd_context(void) : { : bool busy = __this_cpu_xchg(fpsimd_context_busy, false); 0.00 : ffff8000100853a8: adrp x0, ffff80001151d000 0.00 : ffff8000100853ac: add x0, x0, #0x60 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000100853b0: mrs x1, tpidr_el1 : __put_cpu_fpsimd_context(): 18.21 : ffff8000100853b4: add x0, x0, #0x18 6.17 : ffff8000100853b8: ldrb w2, [x0, x1] 71.21 : ffff8000100853bc: strb wzr, [x0, x1] : : WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */ 0.00 : ffff8000100853c0: cbz w2, ffff8000100853c8 <__put_cpu_fpsimd_context+0x20> : } 4.42 : ffff8000100853c4: ret : WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */ 0.00 : ffff8000100853c8: brk #0x800 : } 0.00 : ffff8000100853cc: ret Percent | Source code & Disassembly of vmlinux for cycles (32 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011f9c0 : : set_next_entity(): : resched_curr(rq_of(cfs_rq)); : } : : static void : set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) : { 0.00 : ffff80001011f9c0: stp x29, x30, [sp, #-64]! 0.00 : ffff80001011f9c4: mov x29, sp 6.09 : ffff80001011f9c8: stp x19, x20, [sp, #16] 0.00 : ffff80001011f9cc: mov x20, x0 0.00 : ffff80001011f9d0: mov x19, x1 : /* 'current' is not kept within the tree. */ : if (se->on_rq) { 0.00 : ffff80001011f9d4: ldr w0, [x1, #64] 0.00 : ffff80001011f9d8: cbnz w0, ffff80001011fa00 : update_stats_curr_start(): : se->exec_start = rq_clock_task(rq_of(cfs_rq)); 3.19 : ffff80001011f9dc: ldr x0, [x20, #304] 9.29 : ffff80001011f9e0: ldr x0, [x0, #2432] 12.75 : ffff80001011f9e4: str x0, [x19, #72] : set_next_entity(): : __dequeue_entity(cfs_rq, se); : update_load_avg(cfs_rq, se, UPDATE_TG); : } : : update_stats_curr_start(cfs_rq, se); : cfs_rq->curr = se; 0.00 : ffff80001011f9e8: str x19, [x20, #72] : schedstat_set(se->statistics.slice_max, : max((u64)schedstat_val(se->statistics.slice_max), : se->sum_exec_runtime - se->prev_sum_exec_runtime)); : } : : se->prev_sum_exec_runtime = se->sum_exec_runtime; 0.00 : ffff80001011f9ec: ldr x0, [x19, #80] 0.00 : ffff80001011f9f0: str x0, [x19, #96] : } 3.02 : ffff80001011f9f4: ldp x19, x20, [sp, #16] 0.00 : ffff80001011f9f8: ldp x29, x30, [sp], #64 0.00 : ffff80001011f9fc: ret 44.07 : ffff80001011fa00: str x21, [x29, #32] : __dequeue_entity(): : rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline); 0.00 : ffff80001011fa04: add x21, x19, #0x18 0.00 : ffff80001011fa08: add x1, x20, #0x38 : rb_erase_cached(): : } : : static inline void rb_erase_cached(struct rb_node *node, : struct rb_root_cached *root) : { : if (root->rb_leftmost == node) 9.58 : ffff80001011fa0c: ldr x0, [x20, #64] 0.00 : ffff80001011fa10: cmp x21, x0 0.00 : ffff80001011fa14: b.eq ffff80001011fa38 // b.none : root->rb_leftmost = rb_next(node); : rb_erase(node, &root->rb_root); 0.00 : ffff80001011fa18: mov x0, x21 0.00 : ffff80001011fa1c: bl ffff800010cc4750 : set_next_entity(): : update_load_avg(cfs_rq, se, UPDATE_TG); 0.00 : ffff80001011fa20: mov w2, #0x1 // #1 0.00 : ffff80001011fa24: mov x1, x19 0.00 : ffff80001011fa28: mov x0, x20 0.00 : ffff80001011fa2c: bl ffff80001011e940 0.00 : ffff80001011fa30: ldr x21, [x29, #32] 0.00 : ffff80001011fa34: b ffff80001011f9dc 3.18 : ffff80001011fa38: str x1, [x29, #56] : rb_erase_cached(): : root->rb_leftmost = rb_next(node); 0.00 : ffff80001011fa3c: mov x0, x21 3.02 : ffff80001011fa40: bl ffff800010cc5050 0.00 : ffff80001011fa44: str x0, [x20, #64] 5.81 : ffff80001011fa48: ldr x1, [x29, #56] 0.00 : ffff80001011fa4c: b ffff80001011fa18 Percent | Source code & Disassembly of vmlinux for cycles (49 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010085440 : : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 10.46 : ffff800010085440: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 10.29 : ffff800010085444: ldr w0, [x0, #16] : have_cpu_fpsimd_context(): : preempt_enable(); : } : : static bool have_cpu_fpsimd_context(void) : { : return !preemptible() && __this_cpu_read(fpsimd_context_busy); 0.00 : ffff800010085448: cbz w0, ffff800010085464 64.90 : ffff80001008544c: adrp x0, ffff80001151d000 0.00 : ffff800010085450: add x0, x0, #0x60 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 10.32 : ffff800010085454: mrs x1, tpidr_el1 : have_cpu_fpsimd_context(): 0.00 : ffff800010085458: add x0, x0, #0x18 4.03 : ffff80001008545c: ldrb w0, [x0, x1] : } 0.00 : ffff800010085460: ret : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010085464: mrs x1, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010085468: and w0, w1, #0x80 : have_cpu_fpsimd_context(): : return !preemptible() && __this_cpu_read(fpsimd_context_busy); 0.00 : ffff80001008546c: cbnz w0, ffff80001008544c : } 0.00 : ffff800010085470: ret Percent | Source code & Disassembly of vmlinux for cycles (39 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101511d8 : : irqd_has_set(): : __irqd_to_state(d) |= mask; : } : : static inline bool irqd_has_set(struct irq_data *d, unsigned int mask) : { : return __irqd_to_state(d) & mask; 10.58 : ffff8000101511d8: ldr x2, [x0, #104] 12.83 : ffff8000101511dc: ldr w2, [x2] : irq_may_run(): : : /* : * If the interrupt is not in progress and is not an armed : * wakeup interrupt, proceed. : */ : if (!irqd_has_set(&desc->irq_data, mask)) 0.00 : ffff8000101511e0: tst w2, #0xc0000 0.00 : ffff8000101511e4: b.ne ffff8000101511f4 // b.any : return true; 73.99 : ffff8000101511e8: mov w1, #0x1 // #1 : : /* : * Handle a potential concurrent poll on a different core. : */ : return irq_check_poll(desc); : } 0.00 : ffff8000101511ec: mov w0, w1 2.59 : ffff8000101511f0: ret : { 0.00 : ffff8000101511f4: stp x29, x30, [sp, #-32]! 0.00 : ffff8000101511f8: mov x29, sp 0.00 : ffff8000101511fc: str x19, [sp, #16] 0.00 : ffff800010151200: mov x19, x0 : if (irq_pm_check_wakeup(desc)) 0.00 : ffff800010151204: bl ffff800010157ea0 0.00 : ffff800010151208: tst w0, #0xff : return false; 0.00 : ffff80001015120c: mov w1, #0x0 // #0 : if (irq_pm_check_wakeup(desc)) 0.00 : ffff800010151210: b.ne ffff800010151228 // b.any : irq_check_poll(): : if (!(desc->istate & IRQS_POLL_INPROGRESS)) 0.00 : ffff800010151214: ldr w0, [x19, #172] 0.00 : ffff800010151218: tbz w0, #3, ffff800010151228 : return irq_wait_for_poll(desc); 0.00 : ffff80001015121c: mov x0, x19 0.00 : ffff800010151220: bl ffff800010150500 0.00 : ffff800010151224: and w1, w0, #0xff : irq_may_run(): : } 0.00 : ffff800010151228: mov w0, w1 0.00 : ffff80001015122c: ldr x19, [sp, #16] 0.00 : ffff800010151230: ldp x29, x30, [sp], #32 0.00 : ffff800010151234: ret Percent | Source code & Disassembly of vmlinux for cycles (32 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011e090 : : kick_ilb(): : /* : * Kick a CPU to do the nohz balancing, if it is time for it. We pick any : * idle CPU in the HK_FLAG_MISC housekeeping set (if there is one). : */ : static void kick_ilb(unsigned int flags) : { 0.00 : ffff80001011e090: stp x29, x30, [sp, #-48]! : int ilb_cpu; : : nohz.next_balance++; 0.00 : ffff80001011e094: adrp x1, ffff800011afb000 0.00 : ffff80001011e098: add x1, x1, #0xfc0 : { 0.00 : ffff80001011e09c: mov x29, sp 0.00 : ffff80001011e0a0: stp x19, x20, [sp, #16] 0.00 : ffff80001011e0a4: adrp x20, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001011e0a8: stp x21, x22, [sp, #32] 0.00 : ffff80001011e0ac: mov w22, w0 : find_new_ilb(): : for_each_cpu_and(ilb, nohz.idle_cpus_mask, 0.00 : ffff80001011e0b0: mov w19, #0xffffffff // #-1 0.00 : ffff80001011e0b4: mov x21, x1 : kick_ilb(): : nohz.next_balance++; 0.00 : ffff80001011e0b8: ldr x0, [x1, #40] : find_new_ilb(): : for_each_cpu_and(ilb, nohz.idle_cpus_mask, 0.00 : ffff80001011e0bc: add x20, x20, #0x2f0 : kick_ilb(): : nohz.next_balance++; 3.06 : ffff80001011e0c0: add x0, x0, #0x1 18.66 : ffff80001011e0c4: str x0, [x1, #40] 0.00 : ffff80001011e0c8: b ffff80001011e0d4 : find_new_ilb(): : if (idle_cpu(ilb)) 0.00 : ffff80001011e0cc: bl ffff800010118820 3.08 : ffff80001011e0d0: cbnz w0, ffff80001011e10c : for_each_cpu_and(ilb, nohz.idle_cpus_mask, 0.00 : ffff80001011e0d4: mov w0, #0x4 // #4 0.00 : ffff80001011e0d8: bl ffff800010137e40 0.00 : ffff80001011e0dc: mov x1, x21 0.00 : ffff80001011e0e0: mov x2, x0 0.00 : ffff80001011e0e4: mov w0, w19 0.00 : ffff80001011e0e8: bl ffff800010cb93f0 0.00 : ffff80001011e0ec: ldr w1, [x20] 0.00 : ffff80001011e0f0: mov w19, w0 0.00 : ffff80001011e0f4: cmp w0, w1 0.00 : ffff80001011e0f8: b.cc ffff80001011e0cc // b.lo, b.ul, b.last : kick_ilb(): : * This way we generate a sched IPI on the target CPU which : * is idle. And the softirq performing nohz idle load balance : * will be run before returning from the IPI. : */ : smp_send_reschedule(ilb_cpu); : } 0.00 : ffff80001011e0fc: ldp x19, x20, [sp, #16] 0.00 : ffff80001011e100: ldp x21, x22, [sp, #32] 0.00 : ffff80001011e104: ldp x29, x30, [sp], #48 0.00 : ffff80001011e108: ret : if (ilb_cpu >= nr_cpu_ids) 0.00 : ffff80001011e10c: ldr w0, [x20] 0.00 : ffff80001011e110: cmp w19, w0 0.00 : ffff80001011e114: b.cs ffff80001011e0fc // b.hs, b.nlast : flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 0.00 : ffff80001011e118: adrp x0, ffff800011909000 0.00 : ffff80001011e11c: add x0, x0, #0x928 0.00 : ffff80001011e120: adrp x1, ffff800011528000 0.00 : ffff80001011e124: add x1, x1, #0xe80 0.00 : ffff80001011e128: ldr x0, [x0, w19, sxtw #3] 0.00 : ffff80001011e12c: add x1, x1, x0 0.00 : ffff80001011e130: add x0, x1, #0x30 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011e134: b ffff80001011e160 0.00 : ffff80001011e138: b ffff80001011e160 : __lse_atomic_fetch_or(): : ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC_FETCH_OPS(andnot, ldclr) : ATOMIC_FETCH_OPS(or, ldset) 0.00 : ffff80001011e13c: ldsetal w22, w22, [x0] : kick_ilb(): : if (flags & NOHZ_KICK_MASK) 75.20 : ffff80001011e140: tst x22, #0x3 0.00 : ffff80001011e144: b.ne ffff80001011e0fc // b.any : smp_send_reschedule(ilb_cpu); 0.00 : ffff80001011e148: mov w0, w19 0.00 : ffff80001011e14c: bl ffff800010096ae0 : } 0.00 : ffff80001011e150: ldp x19, x20, [sp, #16] 0.00 : ffff80001011e154: ldp x21, x22, [sp, #32] 0.00 : ffff80001011e158: ldp x29, x30, [sp], #48 0.00 : ffff80001011e15c: ret : __ll_sc_atomic_fetch_or(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(and, and, K) : ATOMIC_OPS(or, orr, K) 0.00 : ffff80001011e160: add x1, x1, #0x30 0.00 : ffff80001011e164: b ffff8000101288f0 0.00 : ffff80001011e168: mov w22, w0 : kick_ilb(): : if (flags & NOHZ_KICK_MASK) 0.00 : ffff80001011e16c: tst x22, #0x3 0.00 : ffff80001011e170: b.eq ffff80001011e148 // b.none : } 0.00 : ffff80001011e174: ldp x19, x20, [sp, #16] 0.00 : ffff80001011e178: ldp x21, x22, [sp, #32] 0.00 : ffff80001011e17c: ldp x29, x30, [sp], #48 0.00 : ffff80001011e180: ret Percent | Source code & Disassembly of vmlinux for cycles (36 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010167188 <__msecs_to_jiffies>: : _msecs_to_jiffies(): : * multiple of HZ, divide with the factor between them, but round : * upwards: : */ : static inline unsigned long _msecs_to_jiffies(const unsigned int m) : { : return (m + (MSEC_PER_SEC / HZ) - 1) / (MSEC_PER_SEC / HZ); 97.86 : ffff800010167188: cmp w0, #0x0 0.00 : ffff80001016718c: mov w0, w0 0.00 : ffff800010167190: add x0, x0, #0x3 0.00 : ffff800010167194: mov x1, #0x3ffffffffffffffe // #4611686018427387902 0.00 : ffff800010167198: asr x0, x0, #2 : __msecs_to_jiffies(): : * Negative value, means infinite timeout: : */ : if ((int)m < 0) : return MAX_JIFFY_OFFSET; : return _msecs_to_jiffies(m); : } 2.14 : ffff80001016719c: csel x0, x0, x1, ge // ge = tcont 0.00 : ffff8000101671a0: ret Percent | Source code & Disassembly of vmlinux for cycles (46 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101226f8 : : update_blocked_averages(): : return p->se.avg.load_avg; : } : #endif : : static void update_blocked_averages(int cpu) : { 0.00 : ffff8000101226f8: stp x29, x30, [sp, #-176]! : bool decayed = false, done = true; : struct rq *rq = cpu_rq(cpu); 0.00 : ffff8000101226fc: adrp x1, ffff800011909000 0.00 : ffff800010122700: add x1, x1, #0x928 : { 0.00 : ffff800010122704: mov x29, sp 0.00 : ffff800010122708: stp x19, x20, [sp, #16] : struct rq *rq = cpu_rq(cpu); 0.00 : ffff80001012270c: adrp x19, ffff800011528000 : { 0.00 : ffff800010122710: stp x21, x22, [sp, #32] : struct rq *rq = cpu_rq(cpu); 0.00 : ffff800010122714: add x19, x19, #0xe80 : { 0.00 : ffff800010122718: stp x23, x24, [sp, #48] : __update_blocked_others(): : *done = false; 0.00 : ffff80001012271c: mov w22, #0x0 // #0 : update_blocked_averages(): : { 0.00 : ffff800010122720: str x27, [sp, #80] : struct rq *rq = cpu_rq(cpu); 0.00 : ffff800010122724: ldr x0, [x1, w0, sxtw #3] 0.00 : ffff800010122728: add x19, x19, x0 : rq_lock_irqsave(): : : static inline void : rq_lock_irqsave(struct rq *rq, struct rq_flags *rf) : __acquires(rq->lock) : { : raw_spin_lock_irqsave(&rq->lock, rf->flags); 0.00 : ffff80001012272c: mov x0, x19 0.00 : ffff800010122730: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff800010122734: str x0, [x29, #112] : update_blocked_averages(): : struct rq_flags rf; : : rq_lock_irqsave(rq, &rf); : update_rq_clock(rq); 0.00 : ffff800010122738: mov x0, x19 0.00 : ffff80001012273c: bl ffff800010114980 : rq_clock_pelt(): : static inline u64 rq_clock_pelt(struct rq *rq) : { : lockdep_assert_held(&rq->lock); : assert_clock_updated(rq); : : return rq->clock_pelt - rq->lost_idle_time; 0.00 : ffff800010122740: ldr x21, [x19, #2440] : __update_blocked_others(): : curr_class = rq->curr->sched_class; 0.00 : ffff800010122744: ldr x1, [x19, #2352] : decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | 0.00 : ffff800010122748: adrp x0, ffff800010d06000 0.00 : ffff80001012274c: add x0, x0, #0x1f8 : rq_clock_pelt(): 0.00 : ffff800010122750: ldr x2, [x19, #2448] : __update_blocked_others(): : curr_class = rq->curr->sched_class; 0.00 : ffff800010122754: ldr x23, [x1, #128] : rq_clock_pelt(): 0.00 : ffff800010122758: sub x21, x21, x2 : __update_blocked_others(): : decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | 0.00 : ffff80001012275c: mov x1, x19 0.00 : ffff800010122760: cmp x23, x0 0.00 : ffff800010122764: mov x0, x21 0.00 : ffff800010122768: cset w2, eq // eq = none 0.00 : ffff80001012276c: bl ffff800010134c40 0.00 : ffff800010122770: mov w20, w0 : update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) | 0.00 : ffff800010122774: adrp x0, ffff800010d06000 0.00 : ffff800010122778: add x0, x0, #0x2c0 0.00 : ffff80001012277c: cmp x23, x0 0.00 : ffff800010122780: mov x1, x19 0.00 : ffff800010122784: cset w2, eq // eq = none 0.00 : ffff800010122788: mov x0, x21 0.00 : ffff80001012278c: bl ffff800010134e10 : decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | 0.00 : ffff800010122790: orr w20, w20, w0 : update_irq_load_avg(rq, 0); 0.00 : ffff800010122794: mov x1, #0x0 // #0 0.00 : ffff800010122798: mov x0, x19 0.00 : ffff80001012279c: bl ffff800010134fe0 : update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) | 0.00 : ffff8000101227a0: orr w20, w20, w0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101227a4: ldr x1, [x19, #2672] : __update_blocked_others(): : decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | 0.00 : ffff8000101227a8: cmp w20, #0x0 0.00 : ffff8000101227ac: cset w0, ne // ne = any 0.00 : ffff8000101227b0: str w0, [x29, #148] : others_have_blocked(): : if (READ_ONCE(rq->avg_rt.util_avg)) 0.00 : ffff8000101227b4: cbnz x1, ffff8000101227c0 : __read_once_size(): 2.19 : ffff8000101227b8: ldr x0, [x19, #2736] : others_have_blocked(): : if (READ_ONCE(rq->avg_dl.util_avg)) 0.00 : ffff8000101227bc: cbz x0, ffff800010122db4 : __update_blocked_fair(): : for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 0.00 : ffff8000101227c0: ldr x21, [x19, #2320] 0.00 : ffff8000101227c4: add x24, x19, #0x910 : int cpu = cpu_of(rq); 0.00 : ffff8000101227c8: ldr w23, [x19, #2568] : for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 0.00 : ffff8000101227cc: sub x27, x21, #0x140 0.00 : ffff8000101227d0: cmp x24, x21 0.00 : ffff8000101227d4: ldr x20, [x27, #320] 0.00 : ffff8000101227d8: sub x20, x20, #0x140 0.00 : ffff8000101227dc: b.eq ffff800010122c28 // b.none : se = cfs_rq->tg->se[cpu]; 16.20 : ffff8000101227e0: sbfiz x23, x23, #3, #32 0.00 : ffff8000101227e4: stp x25, x26, [x29, #64] : cpufreq_update_util(): : */ : static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) : { : struct update_util_data *data; : : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff8000101227e8: adrp x0, ffff80001151f000 : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff8000101227ec: adrp x26, ffff800011afb000 : cpufreq_update_util(): 0.00 : ffff8000101227f0: add x0, x0, #0x158 : update_tg_load_avg(): 0.00 : ffff8000101227f4: add x1, x26, #0xd80 0.00 : ffff8000101227f8: str x28, [x29, #88] : cpufreq_update_util(): 0.00 : ffff8000101227fc: str x0, [x29, #104] : update_tg_load_avg(): 0.00 : ffff800010122800: str x1, [x29, #136] : __update_blocked_fair(): : bool decayed = false; 0.00 : ffff800010122804: str wzr, [x29, #172] : cfs_rq_clock_pelt(): : return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; : } : #else : static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) : { : return rq_clock_pelt(rq_of(cfs_rq)); 0.00 : ffff800010122808: ldr x0, [x27, #304] : update_cfs_rq_load_avg(): : if (cfs_rq->removed.nr) { 2.24 : ffff80001012280c: ldr w25, [x27, #196] : rq_clock_pelt(): : return rq->clock_pelt - rq->lost_idle_time; 3.87 : ffff800010122810: ldr x4, [x0, #2440] 2.47 : ffff800010122814: ldr x0, [x0, #2448] 0.00 : ffff800010122818: sub x26, x4, x0 : update_cfs_rq_load_avg(): 0.00 : ffff80001012281c: cbz w25, ffff8000101228d0 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010122820: ldr w1, [x27, #156] 0.00 : ffff800010122824: mov w3, #0xb67e // #46718 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010122828: add x25, x27, #0xc0 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff80001012282c: add w1, w1, w3 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010122830: mov x0, x25 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010122834: str w1, [x29, #168] : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010122838: bl ffff800010cd80f0 <_raw_spin_lock> : cfs_rq->removed.nr = 0; 0.00 : ffff80001012283c: str wzr, [x27, #196] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff800010122840: ldp x8, x28, [x27, #200] 0.00 : ffff800010122844: stp xzr, xzr, [x27, #200] : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010122848: ldr x10, [x27, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff80001012284c: mov x0, x25 : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010122850: str xzr, [x27, #216] 0.00 : ffff800010122854: stp x8, x10, [x29, #152] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff800010122858: bl ffff800010cd7c38 <_raw_spin_unlock> : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff80001012285c: ldp x8, x10, [x29, #152] : cfs_rq->propagate = 1; 0.00 : ffff800010122860: mov x9, #0x1 // #1 : __read_once_size(): 0.00 : ffff800010122864: ldr x2, [x27, #160] : update_cfs_rq_load_avg(): : decayed = 1; 0.00 : ffff800010122868: mov w25, w9 : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff80001012286c: ldr x6, [x27, #272] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff800010122870: sub x5, x2, x8 : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010122874: ldr w1, [x29, #168] : sub_positive(&sa->load_avg, r); 0.00 : ffff800010122878: cmp x2, x5 : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff80001012287c: str x9, [x27, #264] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff800010122880: csel x5, x5, xzr, cs // cs = hs, nlast : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010122884: str x5, [x27, #160] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010122888: sub x5, x6, x10 : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff80001012288c: mov w0, w1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010122890: ldr x6, [x27, #136] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010122894: str x5, [x27, #272] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010122898: msub x0, x0, x8, x6 0.00 : ffff80001012289c: cmp x6, x0 0.00 : ffff8000101228a0: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101228a4: str x0, [x27, #136] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101228a8: ldr x5, [x27, #176] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_avg, r); 0.00 : ffff8000101228ac: sub x0, x5, x28 0.00 : ffff8000101228b0: cmp x5, x0 0.00 : ffff8000101228b4: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101228b8: str x0, [x27, #176] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101228bc: ldr w5, [x27, #152] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_sum, r * divider); 0.00 : ffff8000101228c0: msub w0, w1, w28, w5 0.00 : ffff8000101228c4: cmp w5, w0 0.00 : ffff8000101228c8: csel w0, w0, wzr, cs // cs = hs, nlast : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101228cc: str w0, [x27, #152] : update_cfs_rq_load_avg(): : decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 0.00 : ffff8000101228d0: mov x1, x27 0.00 : ffff8000101228d4: mov x0, x26 0.00 : ffff8000101228d8: bl ffff800010134a68 <__update_load_avg_cfs_rq> 0.00 : ffff8000101228dc: orr w0, w0, w25 : __update_blocked_fair(): : if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 0.00 : ffff8000101228e0: cbnz w0, ffff800010122cb8 0.00 : ffff8000101228e4: ldr x0, [x27, #336] : se = cfs_rq->tg->se[cpu]; 0.00 : ffff8000101228e8: ldr x0, [x0, #240] 2.35 : ffff8000101228ec: ldr x28, [x0, x23] : if (se && !skip_blocked_update(se)) 0.00 : ffff8000101228f0: cbz x28, ffff800010122bc0 : skip_blocked_update(): : if (se->avg.load_avg || se->avg.util_avg) 0.00 : ffff8000101228f4: ldr x0, [x28, #224] 0.00 : ffff8000101228f8: cbnz x0, ffff800010122910 6.64 : ffff8000101228fc: ldr x0, [x28, #240] 0.00 : ffff800010122900: cbnz x0, ffff800010122910 : if (gcfs_rq->propagate) 0.00 : ffff800010122904: ldr x0, [x28, #136] 2.22 : ffff800010122908: ldr x0, [x0, #264] 0.00 : ffff80001012290c: cbz x0, ffff800010122bc0 : __update_blocked_fair(): : update_load_avg(cfs_rq_of(se), se, 0); 12.33 : ffff800010122910: ldr x26, [x28, #128] : update_load_avg(): : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 6.22 : ffff800010122914: ldr x1, [x28, #192] : cfs_rq_clock_pelt(): : return rq_clock_pelt(rq_of(cfs_rq)); 0.00 : ffff800010122918: ldr x0, [x26, #304] : rq_clock_pelt(): : return rq->clock_pelt - rq->lost_idle_time; 4.77 : ffff80001012291c: ldr x6, [x0, #2440] 0.00 : ffff800010122920: ldr x0, [x0, #2448] 0.00 : ffff800010122924: sub x6, x6, x0 : update_load_avg(): 0.00 : ffff800010122928: cbz x1, ffff800010122944 : __update_load_avg_se(now, cfs_rq, se); 0.00 : ffff80001012292c: mov x0, x6 0.00 : ffff800010122930: str x6, [x29, #160] 0.00 : ffff800010122934: mov x2, x28 0.00 : ffff800010122938: mov x1, x26 0.00 : ffff80001012293c: bl ffff800010134868 <__update_load_avg_se> 0.00 : ffff800010122940: ldr x6, [x29, #160] : update_cfs_rq_load_avg(): : if (cfs_rq->removed.nr) { 0.00 : ffff800010122944: ldr w25, [x26, #196] 0.00 : ffff800010122948: cbz w25, ffff800010122a08 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff80001012294c: ldr w1, [x26, #156] : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010122950: add x9, x26, #0xc0 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010122954: mov w2, #0xb67e // #46718 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010122958: stp x6, x9, [x29, #120] : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff80001012295c: add w1, w1, w2 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010122960: mov x0, x9 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010122964: str w1, [x29, #168] : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010122968: bl ffff800010cd80f0 <_raw_spin_lock> : cfs_rq->removed.nr = 0; 0.00 : ffff80001012296c: str wzr, [x26, #196] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff800010122970: ldp x25, x8, [x26, #200] 0.00 : ffff800010122974: stp xzr, xzr, [x26, #200] : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010122978: ldr x10, [x26, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff80001012297c: ldr x9, [x29, #128] : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010122980: str xzr, [x26, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff800010122984: mov x0, x9 : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010122988: stp x8, x10, [x29, #152] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff80001012298c: bl ffff800010cd7c38 <_raw_spin_unlock> : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010122990: ldp x8, x10, [x29, #152] 0.00 : ffff800010122994: ldr x2, [x26, #272] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010122998: ldr x9, [x26, #160] : add_tg_cfs_propagate(): 0.00 : ffff80001012299c: sub x2, x2, x10 0.00 : ffff8000101229a0: str x2, [x26, #272] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff8000101229a4: sub x2, x9, x25 : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff8000101229a8: ldr w1, [x29, #168] : sub_positive(&sa->load_avg, r); 0.00 : ffff8000101229ac: cmp x9, x2 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101229b0: ldr x6, [x29, #120] : update_cfs_rq_load_avg(): 0.00 : ffff8000101229b4: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101229b8: str x2, [x26, #160] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff8000101229bc: mov w0, w1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101229c0: ldr x2, [x26, #136] : update_cfs_rq_load_avg(): 0.00 : ffff8000101229c4: msub x0, x0, x25, x2 0.00 : ffff8000101229c8: cmp x2, x0 : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff8000101229cc: mov x2, #0x1 // #1 : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff8000101229d0: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101229d4: str x0, [x26, #136] : update_cfs_rq_load_avg(): : decayed = 1; 0.00 : ffff8000101229d8: mov w25, w2 : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff8000101229dc: str x2, [x26, #264] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101229e0: ldr x7, [x26, #176] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_avg, r); 0.00 : ffff8000101229e4: sub x0, x7, x8 0.00 : ffff8000101229e8: cmp x7, x0 0.00 : ffff8000101229ec: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101229f0: str x0, [x26, #176] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101229f4: ldr w7, [x26, #152] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_sum, r * divider); 0.00 : ffff8000101229f8: msub w0, w1, w8, w7 0.00 : ffff8000101229fc: cmp w7, w0 0.00 : ffff800010122a00: csel w0, w0, wzr, cs // cs = hs, nlast : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010122a04: str w0, [x26, #152] : update_cfs_rq_load_avg(): : decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 2.07 : ffff800010122a08: mov x1, x26 0.00 : ffff800010122a0c: mov x0, x6 0.00 : ffff800010122a10: bl ffff800010134a68 <__update_load_avg_cfs_rq> 0.00 : ffff800010122a14: orr w0, w0, w25 : propagate_entity_load_avg(): : if (entity_is_task(se)) 0.00 : ffff800010122a18: ldr x1, [x28, #136] 0.00 : ffff800010122a1c: cbz x1, ffff800010122ca0 : if (!gcfs_rq->propagate) 0.00 : ffff800010122a20: ldr x2, [x1, #264] 0.00 : ffff800010122a24: cbz x2, ffff800010122ca0 : gcfs_rq->propagate = 0; 0.00 : ffff800010122a28: str xzr, [x1, #264] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010122a2c: mov x7, #0x1 // #1 : propagate_entity_load_avg(): : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 0.00 : ffff800010122a30: ldr x6, [x1, #272] : cfs_rq = cfs_rq_of(se); 0.00 : ffff800010122a34: ldr x2, [x28, #128] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010122a38: ldr x0, [x2, #272] 0.00 : ffff800010122a3c: add x0, x0, x6 0.00 : ffff800010122a40: stp x7, x0, [x2, #264] : propagate_entity_load_avg(): : update_tg_cfs_util(cfs_rq, se, gcfs_rq); 0.00 : ffff800010122a44: ldr x6, [x1, #176] : update_tg_cfs_util(): : long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 0.00 : ffff800010122a48: ldr x0, [x28, #240] 0.00 : ffff800010122a4c: sub x0, x6, x0 : if (!delta) 0.00 : ffff800010122a50: cmp x0, #0x0 0.00 : ffff800010122a54: b.eq ffff800010122a88 // b.none : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010122a58: mov w7, #0xba7e // #47742 : se->avg.util_avg = gcfs_rq->avg.util_avg; 0.00 : ffff800010122a5c: str x6, [x28, #240] : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010122a60: mul w6, w6, w7 0.00 : ffff800010122a64: str w6, [x28, #216] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010122a68: ldr x6, [x2, #176] : update_tg_cfs_util(): : add_positive(&cfs_rq->avg.util_avg, delta); 0.00 : ffff800010122a6c: add x0, x0, x6 0.00 : ffff800010122a70: ccmp x6, x0, #0x2, lt // lt = tstop 0.00 : ffff800010122a74: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010122a78: str x0, [x2, #176] : update_tg_cfs_util(): : cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010122a7c: ldr x0, [x2, #176] 0.00 : ffff800010122a80: mul w7, w7, w0 0.00 : ffff800010122a84: str w7, [x2, #152] : update_tg_cfs_runnable(): : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 0.00 : ffff800010122a88: ldr x7, [x1, #272] : if (!runnable_sum) 0.00 : ffff800010122a8c: cmp x7, #0x0 0.00 : ffff800010122a90: b.eq ffff800010122bac // b.none : gcfs_rq->prop_runnable_sum = 0; 0.00 : ffff800010122a94: str xzr, [x1, #272] : if (runnable_sum >= 0) { 0.00 : ffff800010122a98: b.lt ffff800010122d88 // b.tstop : runnable_sum += se->avg.load_sum; 0.00 : ffff800010122a9c: ldr x0, [x28, #200] : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff800010122aa0: mov x1, #0xba7e // #47742 : runnable_sum += se->avg.load_sum; 0.00 : ffff800010122aa4: add x7, x7, x0 : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff800010122aa8: cmp x7, x1 0.00 : ffff800010122aac: csel x7, x7, x1, le : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff800010122ab0: ldr w1, [x28, #216] : div_s64_rem(): : * Return: sets ``*remainder``, then returns dividend / divisor : */ : static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) : { : *remainder = dividend % divisor; : return dividend / divisor; 0.00 : ffff800010122ab4: mov x3, #0x5041 // #20545 : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010122ab8: ldr x6, [x28] : div_s64_rem(): 0.00 : ffff800010122abc: movk x3, #0xafb, lsl #16 : update_tg_cfs_runnable(): : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff800010122ac0: ldr x8, [x28, #224] : div_s64_rem(): 0.00 : ffff800010122ac4: movk x3, #0x400, lsl #32 : update_tg_cfs_runnable(): : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff800010122ac8: lsr w1, w1, #10 : div_s64_rem(): 0.00 : ffff800010122acc: movk x3, #0xafb5, lsl #48 : update_tg_cfs_runnable(): : runnable_sum = max(runnable_sum, running_sum); 0.00 : ffff800010122ad0: cmp x1, x7 : se_weight(): 0.00 : ffff800010122ad4: lsr x6, x6, #10 : update_tg_cfs_runnable(): 0.00 : ffff800010122ad8: csel x7, x1, x7, ge // ge = tcont : se->avg.load_sum = runnable_sum; 0.00 : ffff800010122adc: str x7, [x28, #200] : load_sum = (s64)se_weight(se) * runnable_sum; 0.00 : ffff800010122ae0: mul x1, x6, x7 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 0.00 : ffff800010122ae4: msub x6, x6, x0, x1 : div_s64_rem(): 0.00 : ffff800010122ae8: lsr x0, x1, #1 0.00 : ffff800010122aec: mov x1, #0x5041 // #20545 0.00 : ffff800010122af0: movk x1, #0xafb, lsl #16 0.00 : ffff800010122af4: movk x1, #0x400, lsl #32 0.00 : ffff800010122af8: movk x1, #0xafb5, lsl #48 0.00 : ffff800010122afc: umulh x0, x0, x1 0.00 : ffff800010122b00: lsr x0, x0, #14 : update_tg_cfs_runnable(): : se->avg.load_avg = load_avg; 0.00 : ffff800010122b04: str x0, [x28, #224] : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff800010122b08: sub x0, x0, x8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010122b0c: ldr x1, [x2, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_avg, delta_avg); 0.00 : ffff800010122b10: cmp x0, #0x0 0.00 : ffff800010122b14: add x0, x0, x1 0.00 : ffff800010122b18: ccmp x1, x0, #0x2, lt // lt = tstop 0.00 : ffff800010122b1c: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010122b20: str x0, [x2, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_sum, delta_sum); 0.00 : ffff800010122b24: cmp x6, #0x0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010122b28: ldr x1, [x2, #136] : update_tg_cfs_runnable(): 0.00 : ffff800010122b2c: add x0, x6, x1 0.00 : ffff800010122b30: ccmp x1, x0, #0x2, lt // lt = tstop 0.00 : ffff800010122b34: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010122b38: str x0, [x2, #136] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff800010122b3c: ldr x1, [x28, #16] : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff800010122b40: ldr w6, [x28, #64] : se_runnable(): 0.00 : ffff800010122b44: lsr x1, x1, #10 : update_tg_cfs_runnable(): : runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 0.00 : ffff800010122b48: mul x1, x1, x7 : div_s64_rem(): 0.00 : ffff800010122b4c: lsr x0, x1, #1 0.00 : ffff800010122b50: umulh x0, x0, x3 0.00 : ffff800010122b54: lsr x0, x0, #14 : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff800010122b58: cbz w6, ffff800010122ba4 : delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 0.00 : ffff800010122b5c: ldr x8, [x28, #232] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010122b60: ldr x6, [x2, #168] : update_tg_cfs_runnable(): 0.00 : ffff800010122b64: sub x8, x0, x8 : se_weight(se) * se->avg.runnable_load_sum; 0.00 : ffff800010122b68: ldr x9, [x28, #208] : add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 0.00 : ffff800010122b6c: cmp x8, #0x0 0.00 : ffff800010122b70: add x8, x8, x6 0.00 : ffff800010122b74: ccmp x6, x8, #0x2, lt // lt = tstop : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010122b78: ldr x6, [x28] : update_tg_cfs_runnable(): 0.00 : ffff800010122b7c: csel x8, x8, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010122b80: str x8, [x2, #168] : se_weight(): 0.00 : ffff800010122b84: lsr x6, x6, #10 : update_tg_cfs_runnable(): : delta_sum = runnable_load_sum - 0.00 : ffff800010122b88: msub x1, x6, x9, x1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010122b8c: ldr x6, [x2, #144] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 0.00 : ffff800010122b90: cmp x1, #0x0 0.00 : ffff800010122b94: add x1, x1, x6 0.00 : ffff800010122b98: ccmp x6, x1, #0x2, lt // lt = tstop 0.00 : ffff800010122b9c: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010122ba0: str x1, [x2, #144] : update_tg_cfs_runnable(): : se->avg.runnable_load_sum = runnable_sum; 0.00 : ffff800010122ba4: str x7, [x28, #208] : se->avg.runnable_load_avg = runnable_load_avg; 0.00 : ffff800010122ba8: str x0, [x28, #232] : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010122bac: ldr x1, [x26, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff800010122bb0: add x0, x1, #0x80 0.00 : ffff800010122bb4: cmp x26, x0 0.00 : ffff800010122bb8: b.eq ffff800010122d1c // b.none 0.00 : ffff800010122bbc: nop : cfs_rq_is_decayed(): : if (cfs_rq->load.weight) 8.95 : ffff800010122bc0: ldr x0, [x27] 0.00 : ffff800010122bc4: cbnz x0, ffff800010122be0 : if (cfs_rq->avg.load_sum) 0.00 : ffff800010122bc8: ldr x0, [x27, #136] 0.00 : ffff800010122bcc: cbnz x0, ffff800010122be0 : if (cfs_rq->avg.util_sum) 0.00 : ffff800010122bd0: ldr w0, [x27, #152] 0.00 : ffff800010122bd4: cbnz w0, ffff800010122be0 : if (cfs_rq->avg.runnable_load_sum) 0.00 : ffff800010122bd8: ldr x0, [x27, #144] 0.00 : ffff800010122bdc: cbz x0, ffff800010122d4c : cfs_rq_has_blocked(): : if (cfs_rq->avg.load_avg) 0.00 : ffff800010122be0: ldr x0, [x27, #160] 0.00 : ffff800010122be4: cbnz x0, ffff800010122c98 : if (cfs_rq->avg.util_avg) 0.00 : ffff800010122be8: ldr x0, [x27, #176] : __update_blocked_fair(): : *done = false; 0.00 : ffff800010122bec: cmp x0, #0x0 0.00 : ffff800010122bf0: csel w22, w22, wzr, eq // eq = none : for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 0.00 : ffff800010122bf4: ldr x0, [x20, #320] 0.00 : ffff800010122bf8: add x21, x20, #0x140 0.00 : ffff800010122bfc: mov x27, x20 0.00 : ffff800010122c00: cmp x24, x21 0.00 : ffff800010122c04: sub x20, x0, #0x140 0.00 : ffff800010122c08: b.ne ffff800010122808 // b.any 0.00 : ffff800010122c0c: ldr w1, [x29, #148] 0.00 : ffff800010122c10: ldr w0, [x29, #172] 0.00 : ffff800010122c14: ldp x25, x26, [x29, #64] 0.00 : ffff800010122c18: orr w28, w0, w1 0.00 : ffff800010122c1c: and w0, w28, #0xff 0.00 : ffff800010122c20: ldr x28, [x29, #88] 0.00 : ffff800010122c24: str w0, [x29, #148] : update_blocked_load_status(): : rq->last_blocked_load_update_tick = jiffies; 0.00 : ffff800010122c28: adrp x0, ffff800011907000 0.00 : ffff800010122c2c: ldr x0, [x0, #2432] 0.00 : ffff800010122c30: str x0, [x19, #32] : if (!has_blocked) 0.00 : ffff800010122c34: cbz w22, ffff800010122c3c : rq->has_blocked_load = 0; 0.00 : ffff800010122c38: str wzr, [x19, #40] : update_blocked_averages(): : : decayed |= __update_blocked_others(rq, &done); : decayed |= __update_blocked_fair(rq, &done); : : update_blocked_load_status(rq, !done); : if (decayed) 6.89 : ffff800010122c3c: ldr w0, [x29, #148] 0.00 : ffff800010122c40: cbz w0, ffff800010122c74 : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff800010122c44: ldrsw x1, [x19, #2568] 0.00 : ffff800010122c48: adrp x0, ffff800011909000 0.00 : ffff800010122c4c: add x27, x0, #0x928 0.00 : ffff800010122c50: adrp x0, ffff80001151f000 0.00 : ffff800010122c54: add x0, x0, #0x158 0.00 : ffff800010122c58: ldr x1, [x27, x1, lsl #3] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010122c5c: ldr x0, [x0, x1] : cpufreq_update_util(): : cpu_of(rq))); : if (data) 1.71 : ffff800010122c60: cbz x0, ffff800010122c74 : data->func(data, rq_clock(rq), flags); 0.00 : ffff800010122c64: ldr x3, [x0] 0.00 : ffff800010122c68: mov w2, #0x0 // #0 0.00 : ffff800010122c6c: ldr x1, [x19, #2400] 0.00 : ffff800010122c70: blr x3 : rq_unlock_irqrestore(): : raw_spin_unlock_irqrestore(&rq->lock, rf->flags); 12.46 : ffff800010122c74: ldr x1, [x29, #112] 0.00 : ffff800010122c78: mov x0, x19 0.00 : ffff800010122c7c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : update_blocked_averages(): : cpufreq_update_util(rq, 0); : rq_unlock_irqrestore(rq, &rf); : } 0.00 : ffff800010122c80: ldr x27, [sp, #80] 0.00 : ffff800010122c84: ldp x19, x20, [sp, #16] 0.00 : ffff800010122c88: ldp x21, x22, [sp, #32] 0.00 : ffff800010122c8c: ldp x23, x24, [sp, #48] 0.00 : ffff800010122c90: ldp x29, x30, [sp], #176 0.00 : ffff800010122c94: ret : __update_blocked_fair(): : *done = false; 0.00 : ffff800010122c98: mov w22, #0x0 // #0 0.00 : ffff800010122c9c: b ffff800010122bf4 : update_load_avg(): : } else if (decayed) { 1.94 : ffff800010122ca0: cbz w0, ffff800010122bc0 : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010122ca4: ldr x1, [x26, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff800010122ca8: add x0, x1, #0x80 0.00 : ffff800010122cac: cmp x26, x0 0.00 : ffff800010122cb0: b.ne ffff800010122bc0 // b.any 0.00 : ffff800010122cb4: b ffff800010122d1c : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff800010122cb8: ldr x0, [x27, #336] 2.34 : ffff800010122cbc: ldr x1, [x29, #136] 0.00 : ffff800010122cc0: cmp x0, x1 0.00 : ffff800010122cc4: b.eq ffff800010122d04 // b.none : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 0.00 : ffff800010122cc8: ldr x1, [x27, #160] 0.00 : ffff800010122ccc: ldr x2, [x27, #256] : if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 0.00 : ffff800010122cd0: subs x1, x1, x2 0.00 : ffff800010122cd4: cneg x4, x1, mi // mi = first 0.00 : ffff800010122cd8: cmp x4, x2, lsr #6 0.00 : ffff800010122cdc: b.ls ffff800010122d04 // b.plast : atomic_long_add(delta, &cfs_rq->tg->load_avg); 0.00 : ffff800010122ce0: add x0, x0, #0x140 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010122ce4: b ffff800010122cf4 0.00 : ffff800010122ce8: b ffff800010122cf4 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff800010122cec: stadd x1, [x0] 0.00 : ffff800010122cf0: b ffff800010122cf8 : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff800010122cf4: b ffff800010128a94 0.00 : ffff800010122cf8: ldr x0, [x27, #336] : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff800010122cfc: ldr x1, [x27, #160] 0.00 : ffff800010122d00: str x1, [x27, #256] : __update_blocked_fair(): : if (cfs_rq == &rq->cfs) 0.00 : ffff800010122d04: add x1, x19, #0x80 : decayed = true; 0.00 : ffff800010122d08: cmp x27, x1 0.00 : ffff800010122d0c: ldr w1, [x29, #172] 0.00 : ffff800010122d10: csinc w1, w1, wzr, ne // ne = any 0.00 : ffff800010122d14: str w1, [x29, #172] 0.00 : ffff800010122d18: b ffff8000101228e8 : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff800010122d1c: ldrsw x4, [x1, #2568] 0.00 : ffff800010122d20: adrp x0, ffff800011909000 0.00 : ffff800010122d24: add x2, x0, #0x928 0.00 : ffff800010122d28: ldr x0, [x29, #104] 0.00 : ffff800010122d2c: ldr x2, [x2, x4, lsl #3] : __read_once_size(): 0.00 : ffff800010122d30: ldr x0, [x0, x2] : cpufreq_update_util(): : if (data) 0.00 : ffff800010122d34: cbz x0, ffff800010122bc0 : data->func(data, rq_clock(rq), flags); 0.00 : ffff800010122d38: ldr x4, [x0] 0.00 : ffff800010122d3c: mov w2, #0x0 // #0 0.00 : ffff800010122d40: ldr x1, [x1, #2400] 0.00 : ffff800010122d44: blr x4 0.00 : ffff800010122d48: b ffff800010122bc0 : list_del_leaf_cfs_rq(): : if (cfs_rq->on_list) { 0.00 : ffff800010122d4c: ldr w0, [x27, #312] 0.00 : ffff800010122d50: cbz w0, ffff800010122be0 : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010122d54: ldr x1, [x27, #304] 0.00 : ffff800010122d58: ldr x0, [x27, #328] : if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 0.00 : ffff800010122d5c: ldr x2, [x1, #2336] 0.00 : ffff800010122d60: cmp x21, x2 0.00 : ffff800010122d64: b.eq ffff800010122dc4 // b.none : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff800010122d68: ldr x1, [x27, #320] : list_del_rcu(): : * grace period has elapsed. : */ : static inline void list_del_rcu(struct list_head *entry) : { : __list_del_entry(entry); : entry->prev = LIST_POISON2; 0.00 : ffff800010122d6c: mov x2, #0x122 // #290 0.00 : ffff800010122d70: movk x2, #0xdead, lsl #48 : __list_del(): : next->prev = prev; 0.00 : ffff800010122d74: str x0, [x1, #8] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010122d78: str x1, [x0] : list_del_leaf_cfs_rq(): : cfs_rq->on_list = 0; 0.00 : ffff800010122d7c: str wzr, [x27, #312] : list_del_rcu(): 0.00 : ffff800010122d80: str x2, [x27, #328] 0.00 : ffff800010122d84: b ffff800010122be0 : update_tg_cfs_runnable(): : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010122d88: ldr x0, [x1] : u64 runnable_load_sum, load_sum = 0; 0.00 : ffff800010122d8c: mov x7, #0x0 // #0 : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010122d90: lsr x0, x0, #10 0.00 : ffff800010122d94: cbz x0, ffff800010122da4 : div_s64_rem(): 0.00 : ffff800010122d98: ldr x7, [x1, #136] : *remainder = dividend % divisor; 0.00 : ffff800010122d9c: sxtw x0, w0 : return dividend / divisor; 0.00 : ffff800010122da0: sdiv x7, x7, x0 : update_tg_cfs_runnable(): : runnable_sum = min(se->avg.load_sum, load_sum); 0.00 : ffff800010122da4: ldr x0, [x28, #200] 0.00 : ffff800010122da8: cmp x0, x7 0.00 : ffff800010122dac: csel x7, x0, x7, ls // ls = plast 0.00 : ffff800010122db0: b ffff800010122ab0 : __read_once_size(): : __READ_ONCE_SIZE; 2.13 : ffff800010122db4: ldr x0, [x19, #2800] : others_have_blocked(): : if (READ_ONCE(rq->avg_irq.util_avg)) 0.00 : ffff800010122db8: cmp x0, #0x0 0.00 : ffff800010122dbc: cset w22, eq // eq = none 0.00 : ffff800010122dc0: b ffff8000101227c0 : list_del_leaf_cfs_rq(): : rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 0.00 : ffff800010122dc4: str x0, [x1, #2336] 0.00 : ffff800010122dc8: ldr x0, [x27, #328] 0.00 : ffff800010122dcc: b ffff800010122d68 Percent | Source code & Disassembly of vmlinux for cycles (60 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015df30 : : rcu_preempt_need_deferred_qs(): : * is disabled. This function cannot be expected to understand these : * nuances, so the caller must handle them. : */ : static bool rcu_preempt_need_deferred_qs(struct task_struct *t) : { : return (__this_cpu_read(rcu_data.exp_deferred_qs) || 13.23 : ffff80001015df30: adrp x1, ffff800011529000 0.00 : ffff80001015df34: add x1, x1, #0xac0 0.00 : ffff80001015df38: add x1, x1, #0x15 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 1.71 : ffff80001015df3c: mrs x2, tpidr_el1 : rcu_preempt_need_deferred_qs(): 1.77 : ffff80001015df40: ldrb w1, [x1, x2] : READ_ONCE(t->rcu_read_unlock_special.s)) && 1.66 : ffff80001015df44: cbnz w1, ffff80001015df50 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 55.18 : ffff80001015df48: ldr w0, [x0, #780] : rcu_preempt_need_deferred_qs(): : return (__this_cpu_read(rcu_data.exp_deferred_qs) || 0.00 : ffff80001015df4c: cbz w0, ffff80001015df60 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001015df50: mrs x0, sp_el0 : rcu_preempt_need_deferred_qs(): : READ_ONCE(t->rcu_read_unlock_special.s)) && 0.00 : ffff80001015df54: ldr w0, [x0, #776] 0.00 : ffff80001015df58: cmp w0, #0x0 0.00 : ffff80001015df5c: cset w1, le : rcu_preempt_depth() <= 0; : } 26.46 : ffff80001015df60: mov w0, w1 0.00 : ffff80001015df64: ret Percent | Source code & Disassembly of vmlinux for cycles (92 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010467790 : : blk_mq_make_request(): : * request creation. : * : * Returns: Request queue cookie. : */ : static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) : { 0.00 : ffff800010467790: stp x29, x30, [sp, #-144]! 0.00 : ffff800010467794: mov x29, sp 0.00 : ffff800010467798: str x19, [sp, #16] 0.00 : ffff80001046779c: mov x19, x0 0.00 : ffff8000104677a0: stp x21, x22, [sp, #32] 0.00 : ffff8000104677a4: adrp x21, ffff800011909000 0.00 : ffff8000104677a8: str x23, [sp, #48] 0.00 : ffff8000104677ac: add x2, x21, #0x908 0.00 : ffff8000104677b0: str x1, [x29, #72] 0.00 : ffff8000104677b4: mov w23, #0x1 // #1 1.13 : ffff8000104677b8: ldr x3, [x2] 1.10 : ffff8000104677bc: str x3, [x29, #136] 0.00 : ffff8000104677c0: mov x3, #0x0 // #0 : const int is_sync = op_is_sync(bio->bi_opf); 0.00 : ffff8000104677c4: ldr w22, [x1, #16] : op_is_sync(): : * PREFLUSH flag. Other operations may be marked as synchronous using the : * REQ_SYNC flag. : */ : static inline bool op_is_sync(unsigned int op) : { : return (op & REQ_OP_MASK) == REQ_OP_READ || 0.00 : ffff8000104677c8: and w0, w22, #0xff 0.00 : ffff8000104677cc: cbz w0, ffff8000104677e0 0.00 : ffff8000104677d0: and w0, w22, #0x7f800 0.00 : ffff8000104677d4: and w0, w0, #0xfffe0fff 0.00 : ffff8000104677d8: cmp w0, #0x0 0.00 : ffff8000104677dc: cset w23, ne // ne = any : blk_mq_make_request(): : struct blk_plug *plug; : struct request *same_queue_rq = NULL; : unsigned int nr_segs; : blk_qc_t cookie; : : blk_queue_bounce(q, &bio); 1.01 : ffff8000104677e0: mov x0, x19 0.00 : ffff8000104677e4: add x1, x29, #0x48 : struct request *same_queue_rq = NULL; 0.00 : ffff8000104677e8: str xzr, [x29, #88] : struct blk_mq_alloc_data data = { .flags = 0}; 0.00 : ffff8000104677ec: stp xzr, xzr, [x29, #96] 0.00 : ffff8000104677f0: stp xzr, xzr, [x29, #112] 0.00 : ffff8000104677f4: str xzr, [x29, #128] : blk_queue_bounce(q, &bio); 0.00 : ffff8000104677f8: bl ffff800010476d18 : __blk_queue_split(q, &bio, &nr_segs); 0.00 : ffff8000104677fc: add x2, x29, #0x50 0.00 : ffff800010467800: add x1, x29, #0x48 0.00 : ffff800010467804: mov x0, x19 0.00 : ffff800010467808: bl ffff800010461418 <__blk_queue_split> : : if (!bio_integrity_prep(bio)) 0.00 : ffff80001046780c: ldr x0, [x29, #72] 0.00 : ffff800010467810: bl ffff8000104802c0 0.00 : ffff800010467814: tst w0, #0xff 0.00 : ffff800010467818: b.eq ffff8000104679e8 // b.none : return BLK_QC_T_NONE; : : if (!is_flush_fua && !blk_queue_nomerges(q) && 0.00 : ffff80001046781c: ands w22, w22, #0x60000 1.08 : ffff800010467820: ldr w2, [x29, #80] 0.00 : ffff800010467824: ldr x1, [x29, #72] 0.00 : ffff800010467828: b.ne ffff800010467838 // b.any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001046782c: ldr x0, [x19, #104] : blk_mq_make_request(): 0.00 : ffff800010467830: tst w0, #0x8 0.00 : ffff800010467834: b.eq ffff800010467a00 // b.none : test_bit(): 0.00 : ffff800010467838: ldr x0, [x19, #104] : blk_mq_sched_bio_merge(): : : static inline bool : blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio, : unsigned int nr_segs) : { : if (blk_queue_nomerges(q) || !bio_mergeable(bio)) 0.00 : ffff80001046783c: tst w0, #0x8 0.00 : ffff800010467840: b.ne ffff800010467854 // b.any : bio_mergeable(): : bio_op(bio) == REQ_OP_WRITE_ZEROES; : } : : static inline bool bio_mergeable(struct bio *bio) : { : if (bio->bi_opf & REQ_NOMERGE_FLAGS) 0.00 : ffff800010467844: ldr w0, [x1, #16] 0.00 : ffff800010467848: and w0, w0, #0x7c000 0.00 : ffff80001046784c: and w0, w0, #0xfffe7fff 0.00 : ffff800010467850: cbz w0, ffff800010467a2c 0.00 : ffff800010467854: str x20, [x29, #24] : bio_set_flag(): : return (bio->bi_flags & (1U << bit)) != 0; : } : : static inline void bio_set_flag(struct bio *bio, unsigned int bit) : { : bio->bi_flags |= (1U << bit); 0.00 : ffff800010467858: ldrh w0, [x1, #20] 0.00 : ffff80001046785c: orr w0, w0, #0x1000 1.02 : ffff800010467860: strh w0, [x1, #20] : rq_qos_throttle(): : /* : * BIO_TRACKED lets controllers know that a bio went through the : * normal rq_qos path. : */ : bio_set_flag(bio, BIO_TRACKED); : if (q->rq_qos) 0.00 : ffff800010467864: ldr x0, [x19, #24] 0.00 : ffff800010467868: cbz x0, ffff800010467874 : __rq_qos_throttle(q->rq_qos, bio); 0.00 : ffff80001046786c: bl ffff800010476248 <__rq_qos_throttle> 0.00 : ffff800010467870: ldr x1, [x29, #72] : blk_mq_make_request(): : if (blk_mq_sched_bio_merge(q, bio, nr_segs)) : return BLK_QC_T_NONE; : : rq_qos_throttle(q, bio); : : data.cmd_flags = bio->bi_opf; 0.00 : ffff800010467874: ldr w0, [x1, #16] : rq = blk_mq_get_request(q, bio, &data); 0.00 : ffff800010467878: add x2, x29, #0x60 : data.cmd_flags = bio->bi_opf; 0.00 : ffff80001046787c: str w0, [x29, #112] : rq = blk_mq_get_request(q, bio, &data); 0.00 : ffff800010467880: mov x0, x19 0.00 : ffff800010467884: bl ffff8000104662c8 0.00 : ffff800010467888: mov x20, x0 : if (unlikely(!rq)) { : rq_qos_cleanup(q, bio); 1.11 : ffff80001046788c: ldr x0, [x19, #24] : if (unlikely(!rq)) { 0.00 : ffff800010467890: cbz x20, ffff8000104679c8 : if (bio->bi_opf & REQ_NOWAIT) : bio_wouldblock_error(bio); : return BLK_QC_T_NONE; : } : : trace_block_getrq(q, bio, bio->bi_opf); 0.00 : ffff800010467894: ldr x2, [x29, #72] : rq_qos_track(): : } : : static inline void rq_qos_track(struct request_queue *q, struct request *rq, : struct bio *bio) : { : if (q->rq_qos) 0.00 : ffff800010467898: cbz x0, ffff8000104678a8 : __rq_qos_track(q->rq_qos, rq, bio); 0.00 : ffff80001046789c: mov x1, x20 0.00 : ffff8000104678a0: bl ffff800010476290 <__rq_qos_track> 0.00 : ffff8000104678a4: ldr x2, [x29, #72] : blk_mq_make_request(): : : rq_qos_track(q, rq, bio); : : cookie = request_to_qc_t(data.hctx, rq); 0.00 : ffff8000104678a8: ldr x1, [x29, #128] 0.00 : ffff8000104678ac: ldr w0, [x20, #32] 0.00 : ffff8000104678b0: ldr w1, [x1, #428] : request_to_qc_t(): : ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) : : static inline blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, : struct request *rq) : { : if (rq->tag != -1) 0.00 : ffff8000104678b4: cmn w0, #0x1 0.00 : ffff8000104678b8: lsl w3, w1, #16 0.00 : ffff8000104678bc: b.eq ffff800010467a7c // b.none : return rq->tag | (hctx->queue_num << BLK_QC_T_SHIFT); 0.00 : ffff8000104678c0: orr w0, w0, w3 : blk_mq_bio_to_request(): : if (bio->bi_opf & REQ_RAHEAD) 0.00 : ffff8000104678c4: ldr w3, [x2, #16] : blk_mq_make_request(): : cookie = request_to_qc_t(data.hctx, rq); 3.33 : ffff8000104678c8: str w0, [x29, #84] : : blk_mq_bio_to_request(rq, bio, nr_segs); 0.00 : ffff8000104678cc: ldr w1, [x29, #80] : blk_mq_bio_to_request(): : if (bio->bi_opf & REQ_RAHEAD) 0.00 : ffff8000104678d0: tbnz w3, #19, ffff8000104679f0 : rq->__sector = bio->bi_iter.bi_sector; 0.00 : ffff8000104678d4: ldr x0, [x2, #32] 0.00 : ffff8000104678d8: str x0, [x20, #48] : rq->write_hint = bio->bi_write_hint; 0.00 : ffff8000104678dc: ldrh w0, [x2, #24] : blk_rq_bio_prep(): : } : : static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio, : unsigned int nr_segs) : { : rq->nr_phys_segments = nr_segs; 0.00 : ffff8000104678e0: strh w1, [x20, #194] : blk_mq_bio_to_request(): 1.10 : ffff8000104678e4: strh w0, [x20, #198] : blk_rq_bio_prep(): : rq->__data_len = bio->bi_iter.bi_size; 2.10 : ffff8000104678e8: ldr w0, [x2, #40] 0.00 : ffff8000104678ec: str w0, [x20, #40] : rq->bio = rq->biotail = bio; 1.11 : ffff8000104678f0: stp x2, x2, [x20, #56] : rq->ioprio = bio_prio(bio); 2.10 : ffff8000104678f4: ldrh w0, [x2, #22] 0.00 : ffff8000104678f8: strh w0, [x20, #200] : : if (bio->bi_disk) 4.34 : ffff8000104678fc: ldr x0, [x2, #8] 0.00 : ffff800010467900: cbz x0, ffff800010467908 : rq->rq_disk = bio->bi_disk; 1.10 : ffff800010467904: str x0, [x20, #160] : blk_mq_bio_to_request(): : blk_account_io_start(rq, true); 0.00 : ffff800010467908: mov x0, x20 0.00 : ffff80001046790c: mov w1, #0x1 // #1 0.00 : ffff800010467910: bl ffff80001045c3d8 : blk_queue_is_zoned(): : return q->limits.zoned; : } : : static inline bool blk_queue_is_zoned(struct request_queue *q) : { : switch (blk_queue_zoned_model(q)) { 1.13 : ffff800010467914: ldr w0, [x19, #1140] 0.00 : ffff800010467918: sub w0, w0, #0x1 0.00 : ffff80001046791c: cmp w0, #0x1 0.00 : ffff800010467920: b.hi ffff800010467954 // b.pmore : blk_mq_plug(): : { : /* : * For regular block devices or read operations, use the context plug : * which may be NULL if blk_start_plug() was not executed. : */ : if (!blk_queue_is_zoned(q) || !op_is_write(bio_op(bio))) 0.00 : ffff800010467924: ldr x0, [x29, #72] : op_is_write(): : return (op & 1); 0.00 : ffff800010467928: ldr w0, [x0, #16] : blk_mq_plug(): 0.00 : ffff80001046792c: tbz w0, #0, ffff800010467954 : blk_mq_make_request(): : : plug = blk_mq_plug(q, bio); : if (unlikely(is_flush_fua)) { 0.00 : ffff800010467930: cbnz w22, ffff800010467b14 : blk_flush_plug_list(plug, false); : trace_block_plug(q); : } : : blk_add_rq_to_plug(plug, rq); : } else if (q->elevator) { 0.00 : ffff800010467934: ldr x0, [x19, #8] 0.00 : ffff800010467938: cbz x0, ffff800010467b64 : * to the hardware. : */ : blk_mq_try_issue_directly(data.hctx, rq, &cookie); : } else { : /* Default case. */ : blk_mq_sched_insert_request(rq, false, true, true); 0.00 : ffff80001046793c: mov w3, #0x1 // #1 0.00 : ffff800010467940: mov w1, #0x0 // #0 0.00 : ffff800010467944: mov w2, w3 0.00 : ffff800010467948: mov x0, x20 0.00 : ffff80001046794c: bl ffff80001046cd00 0.00 : ffff800010467950: b ffff800010467998 5.50 : ffff800010467954: str x24, [x29, #56] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 3.33 : ffff800010467958: mrs x0, sp_el0 : blk_mq_plug(): : return current->plug; 0.00 : ffff80001046795c: ldr x24, [x0, #1840] : blk_mq_make_request(): : if (unlikely(is_flush_fua)) { 0.00 : ffff800010467960: cbnz w22, ffff800010467b10 : } else if (plug && (q->nr_hw_queues == 1 || q->mq_ops->commit_rqs || 3.29 : ffff800010467964: cbz x24, ffff800010467b2c 0.00 : ffff800010467968: ldr w0, [x19, #80] 0.00 : ffff80001046796c: cmp w0, #0x1 0.00 : ffff800010467970: b.eq ffff800010467980 // b.none 0.00 : ffff800010467974: ldr x1, [x19, #48] 0.00 : ffff800010467978: ldr x1, [x1, #8] 0.00 : ffff80001046797c: cbz x1, ffff800010467a8c : unsigned int request_count = plug->rq_count; 30.20 : ffff800010467980: ldrh w0, [x24, #32] : if (!request_count) 0.00 : ffff800010467984: cbnz w0, ffff800010467a48 : blk_add_rq_to_plug(plug, rq); 2.21 : ffff800010467988: mov x0, x24 0.00 : ffff80001046798c: mov x1, x20 0.00 : ffff800010467990: bl ffff8000104647c0 : !blk_queue_nonrot(q))) { 2.20 : ffff800010467994: ldr x24, [x29, #56] : } : : return cookie; 12.04 : ffff800010467998: ldr w0, [x29, #84] 0.00 : ffff80001046799c: ldr x20, [x29, #24] : } 0.00 : ffff8000104679a0: add x21, x21, #0x908 0.00 : ffff8000104679a4: ldr x2, [x29, #136] 0.00 : ffff8000104679a8: ldr x1, [x21] 0.00 : ffff8000104679ac: eor x1, x2, x1 0.00 : ffff8000104679b0: cbnz x1, ffff800010467b98 0.00 : ffff8000104679b4: ldr x19, [sp, #16] 3.26 : ffff8000104679b8: ldp x21, x22, [sp, #32] 0.00 : ffff8000104679bc: ldr x23, [sp, #48] 0.00 : ffff8000104679c0: ldp x29, x30, [sp], #144 0.00 : ffff8000104679c4: ret : rq_qos_cleanup(q, bio); 0.00 : ffff8000104679c8: ldr x1, [x29, #72] : rq_qos_cleanup(): : if (q->rq_qos) 0.00 : ffff8000104679cc: cbz x0, ffff8000104679d8 : __rq_qos_cleanup(q->rq_qos, bio); 0.00 : ffff8000104679d0: bl ffff800010476128 <__rq_qos_cleanup> 0.00 : ffff8000104679d4: ldr x1, [x29, #72] : blk_mq_make_request(): : if (bio->bi_opf & REQ_NOWAIT) 0.00 : ffff8000104679d8: ldr w0, [x1, #16] 0.00 : ffff8000104679dc: tbnz w0, #21, ffff800010467b6c 0.00 : ffff8000104679e0: ldr x20, [x29, #24] 0.00 : ffff8000104679e4: nop : return BLK_QC_T_NONE; 0.00 : ffff8000104679e8: mov w0, #0xffffffff // #-1 0.00 : ffff8000104679ec: b ffff8000104679a0 : blk_mq_bio_to_request(): : rq->cmd_flags |= REQ_FAILFAST_MASK; 0.00 : ffff8000104679f0: ldr w0, [x20, #24] 0.00 : ffff8000104679f4: orr w0, w0, #0x700 0.00 : ffff8000104679f8: str w0, [x20, #24] 0.00 : ffff8000104679fc: b ffff8000104678d4 : blk_mq_make_request(): : blk_attempt_plug_merge(q, bio, nr_segs, &same_queue_rq)) 0.00 : ffff800010467a00: add x3, x29, #0x58 0.00 : ffff800010467a04: mov x0, x19 0.00 : ffff800010467a08: bl ffff80001045c978 : if (!is_flush_fua && !blk_queue_nomerges(q) && 0.00 : ffff800010467a0c: tst w0, #0xff 0.00 : ffff800010467a10: b.ne ffff8000104679e8 // b.any : test_bit(): 0.00 : ffff800010467a14: ldr x0, [x19, #104] 3.32 : ffff800010467a18: ldr w2, [x29, #80] 1.10 : ffff800010467a1c: ldr x1, [x29, #72] : blk_mq_sched_bio_merge(): 0.00 : ffff800010467a20: tst w0, #0x8 0.00 : ffff800010467a24: b.eq ffff800010467844 // b.none 0.00 : ffff800010467a28: b ffff800010467854 : return false; : : return __blk_mq_sched_bio_merge(q, bio, nr_segs); 8.58 : ffff800010467a2c: mov x0, x19 0.00 : ffff800010467a30: bl ffff80001046cbc0 <__blk_mq_sched_bio_merge> : blk_mq_make_request(): : if (blk_mq_sched_bio_merge(q, bio, nr_segs)) 2.22 : ffff800010467a34: tst w0, #0xff 0.00 : ffff800010467a38: b.ne ffff8000104679e8 // b.any 0.00 : ffff800010467a3c: ldr x1, [x29, #72] 0.00 : ffff800010467a40: str x20, [x29, #24] 0.00 : ffff800010467a44: b ffff800010467858 : if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 0.00 : ffff800010467a48: cmp w0, #0xf 0.00 : ffff800010467a4c: b.hi ffff800010467a6c // b.pmore : last = list_entry_rq(plug->mq_list.prev); 0.00 : ffff800010467a50: ldr x0, [x24, #8] : if (request_count >= BLK_MAX_REQUEST_COUNT || (last && 0.00 : ffff800010467a54: cmp x0, #0x48 0.00 : ffff800010467a58: b.eq ffff800010467988 // b.none 0.00 : ffff800010467a5c: ldur w1, [x0, #-32] 0.00 : ffff800010467a60: mov w0, #0x1ffff // #131071 0.00 : ffff800010467a64: cmp w1, w0 0.00 : ffff800010467a68: b.ls ffff800010467988 // b.plast : blk_flush_plug_list(plug, false); 0.00 : ffff800010467a6c: mov w1, #0x0 // #0 0.00 : ffff800010467a70: mov x0, x24 0.00 : ffff800010467a74: bl ffff80001045cc00 0.00 : ffff800010467a78: b ffff800010467988 : request_to_qc_t(): : : return rq->internal_tag | (hctx->queue_num << BLK_QC_T_SHIFT) | 0.00 : ffff800010467a7c: ldr w1, [x20, #36] 0.00 : ffff800010467a80: orr w0, w3, #0x80000000 0.00 : ffff800010467a84: orr w0, w0, w1 0.00 : ffff800010467a88: b ffff8000104678c4 : test_bit(): 0.00 : ffff800010467a8c: ldr x1, [x19, #104] : blk_mq_make_request(): : } else if (plug && (q->nr_hw_queues == 1 || q->mq_ops->commit_rqs || 0.00 : ffff800010467a90: tst w1, #0x40 0.00 : ffff800010467a94: b.eq ffff800010467980 // b.none : } else if (q->elevator) { 0.00 : ffff800010467a98: ldr x1, [x19, #8] 0.00 : ffff800010467a9c: cbnz x1, ffff800010467ba4 : test_bit(): 0.00 : ffff800010467aa0: ldr x1, [x19, #104] : blk_mq_make_request(): : } else if (plug && !blk_queue_nomerges(q)) { 0.00 : ffff800010467aa4: tst w1, #0x8 0.00 : ffff800010467aa8: b.ne ffff800010467b34 // b.any : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010467aac: ldr x0, [x24] : blk_mq_make_request(): : if (list_empty(&plug->mq_list)) 0.00 : ffff800010467ab0: cmp x24, x0 0.00 : ffff800010467ab4: b.eq ffff800010467b90 // b.none : if (same_queue_rq) { 0.00 : ffff800010467ab8: ldr x0, [x29, #88] 0.00 : ffff800010467abc: cbz x0, ffff800010467ae4 : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff800010467ac0: ldp x3, x2, [x0, #72] : __list_del(): : next->prev = prev; 0.00 : ffff800010467ac4: str x2, [x3, #8] : blk_mq_make_request(): : list_del_init(&same_queue_rq->queuelist); 0.00 : ffff800010467ac8: add x1, x0, #0x48 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010467acc: str x3, [x2] 0.00 : ffff800010467ad0: str x1, [x0, #72] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010467ad4: str x1, [x0, #80] : blk_mq_make_request(): : plug->rq_count--; 0.00 : ffff800010467ad8: ldrh w0, [x24, #32] 0.00 : ffff800010467adc: sub w0, w0, #0x1 0.00 : ffff800010467ae0: strh w0, [x24, #32] : blk_add_rq_to_plug(plug, rq); 0.00 : ffff800010467ae4: mov x1, x20 0.00 : ffff800010467ae8: mov x0, x24 0.00 : ffff800010467aec: bl ffff8000104647c0 : if (same_queue_rq) { 0.00 : ffff800010467af0: ldr x1, [x29, #88] 0.00 : ffff800010467af4: cbz x1, ffff800010467b88 : data.hctx = same_queue_rq->mq_hctx; 0.00 : ffff800010467af8: ldr x0, [x1, #16] : blk_mq_try_issue_directly(data.hctx, same_queue_rq, 0.00 : ffff800010467afc: add x2, x29, #0x54 : data.hctx = same_queue_rq->mq_hctx; 0.00 : ffff800010467b00: str x0, [x29, #128] : blk_mq_try_issue_directly(data.hctx, same_queue_rq, 0.00 : ffff800010467b04: bl ffff8000104676d0 0.00 : ffff800010467b08: ldr x24, [x29, #56] 0.00 : ffff800010467b0c: b ffff800010467998 0.00 : ffff800010467b10: ldr x24, [x29, #56] : blk_insert_flush(rq); 0.00 : ffff800010467b14: mov x0, x20 0.00 : ffff800010467b18: bl ffff80001045ede8 : blk_mq_run_hw_queue(data.hctx, true); 0.00 : ffff800010467b1c: ldr x0, [x29, #128] 0.00 : ffff800010467b20: mov w1, #0x1 // #1 0.00 : ffff800010467b24: bl ffff800010465418 0.00 : ffff800010467b28: b ffff800010467998 0.00 : ffff800010467b2c: ldr x24, [x29, #56] 0.00 : ffff800010467b30: b ffff800010467934 0.00 : ffff800010467b34: ldr x24, [x29, #56] : } else if ((q->nr_hw_queues > 1 && is_sync) || 0.00 : ffff800010467b38: cmp w0, #0x1 0.00 : ffff800010467b3c: ldr x0, [x29, #128] 0.00 : ffff800010467b40: cset w1, hi // hi = pmore 0.00 : ffff800010467b44: tst w1, w23 0.00 : ffff800010467b48: b.ne ffff800010467b54 // b.any 0.00 : ffff800010467b4c: ldr w1, [x0, #264] 0.00 : ffff800010467b50: cbnz w1, ffff80001046793c : blk_mq_try_issue_directly(data.hctx, rq, &cookie); 0.00 : ffff800010467b54: add x2, x29, #0x54 0.00 : ffff800010467b58: mov x1, x20 0.00 : ffff800010467b5c: bl ffff8000104676d0 0.00 : ffff800010467b60: b ffff800010467998 0.00 : ffff800010467b64: ldr w0, [x19, #80] 0.00 : ffff800010467b68: b ffff800010467b38 : bio_wouldblock_error(): : bio_endio(bio); : } : : static inline void bio_wouldblock_error(struct bio *bio) : { : bio->bi_status = BLK_STS_AGAIN; 0.00 : ffff800010467b6c: mov w0, #0xc // #12 0.00 : ffff800010467b70: strb w0, [x1, #26] : bio_endio(bio); 0.00 : ffff800010467b74: mov x0, x1 0.00 : ffff800010467b78: bl ffff800010456b18 0.00 : ffff800010467b7c: ldr x20, [x29, #24] : blk_mq_make_request(): : return BLK_QC_T_NONE; 0.00 : ffff800010467b80: mov w0, #0xffffffff // #-1 0.00 : ffff800010467b84: b ffff8000104679a0 0.00 : ffff800010467b88: ldr x24, [x29, #56] 0.00 : ffff800010467b8c: b ffff800010467998 : same_queue_rq = NULL; 0.00 : ffff800010467b90: str xzr, [x29, #88] 0.00 : ffff800010467b94: b ffff800010467ae4 0.00 : ffff800010467b98: str x20, [x29, #24] 0.00 : ffff800010467b9c: str x24, [x29, #56] : } 0.00 : ffff800010467ba0: bl ffff8000100e64d0 <__stack_chk_fail> 0.00 : ffff800010467ba4: ldr x24, [x29, #56] 0.00 : ffff800010467ba8: b ffff80001046793c Percent | Source code & Disassembly of vmlinux for cycles (92 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001027a3a8 : : rw_verify_area(): : { : struct inode *inode; : int retval = -EINVAL; : : inode = file_inode(file); : if (unlikely((ssize_t) count < 0)) 0.00 : ffff80001027a3a8: tbnz x3, #63, ffff80001027a404 : { 0.00 : ffff80001027a3ac: stp x29, x30, [sp, #-32]! 0.00 : ffff80001027a3b0: mov x29, sp 3.10 : ffff80001027a3b4: stp x19, x20, [sp, #16] 0.00 : ffff80001027a3b8: mov x19, x1 0.00 : ffff80001027a3bc: mov w20, w0 : : /* : * ranged mandatory locking does not apply to streams - it makes sense : * only for files where position has a meaning. : */ : if (ppos) { 0.00 : ffff80001027a3c0: cbz x2, ffff80001027a3e0 : loff_t pos = *ppos; 1.09 : ffff80001027a3c4: ldr x2, [x2] : : if (unlikely(pos < 0)) { 0.00 : ffff80001027a3c8: tbnz x2, #63, ffff80001027a40c : if (!unsigned_offsets(file)) : return retval; : if (count >= -pos) /* both values are in 0..LLONG_MAX */ : return -EOVERFLOW; : } else if (unlikely((loff_t) (pos + count) < 0)) { 8.79 : ffff80001027a3cc: cmn x2, x3 0.00 : ffff80001027a3d0: b.mi ffff80001027a468 // b.first : inode = file_inode(file); 1.09 : ffff80001027a3d4: ldr x0, [x19, #32] : if (!unsigned_offsets(file)) : return retval; : } : : if (unlikely(inode->i_flctx && mandatory_lock(inode))) { 1.09 : ffff80001027a3d8: ldr x1, [x0, #360] 0.00 : ffff80001027a3dc: cbnz x1, ffff80001027a428 : if (retval < 0) : return retval; : } : } : : return security_file_permission(file, 22.80 : ffff80001027a3e0: cmp w20, #0x0 0.00 : ffff80001027a3e4: mov w2, #0x4 // #4 0.00 : ffff80001027a3e8: mov w1, #0x2 // #2 0.00 : ffff80001027a3ec: mov x0, x19 3.10 : ffff80001027a3f0: csel w1, w2, w1, eq // eq = none 0.00 : ffff80001027a3f4: bl ffff800010439108 : read_write == READ ? MAY_READ : MAY_WRITE); : } 0.00 : ffff80001027a3f8: ldp x19, x20, [sp, #16] 0.00 : ffff80001027a3fc: ldp x29, x30, [sp], #32 0.00 : ffff80001027a400: ret : return retval; 0.00 : ffff80001027a404: mov w0, #0xffffffea // #-22 : } 0.00 : ffff80001027a408: ret : unsigned_offsets(): : return file->f_mode & FMODE_UNSIGNED_OFFSET; 0.00 : ffff80001027a40c: ldr w0, [x1, #68] : rw_verify_area(): : if (!unsigned_offsets(file)) 0.00 : ffff80001027a410: tbz w0, #13, ffff80001027a470 : if (count >= -pos) /* both values are in 0..LLONG_MAX */ 0.00 : ffff80001027a414: neg x0, x2 0.00 : ffff80001027a418: cmp x0, x3 0.00 : ffff80001027a41c: b.hi ffff80001027a3d4 // b.pmore : return -EOVERFLOW; 0.00 : ffff80001027a420: mov w0, #0xffffffb5 // #-75 0.00 : ffff80001027a424: b ffff80001027a3f8 : mandatory_lock(): : * otherwise these will be advisory locks : */ : : static inline int mandatory_lock(struct inode *ino) : { : return IS_MANDLOCK(ino) && __mandatory_lock(ino); 7.69 : ffff80001027a428: ldr x1, [x0, #40] 51.26 : ffff80001027a42c: ldr x1, [x1, #80] 0.00 : ffff80001027a430: tbz w1, #6, ffff80001027a3e0 0.00 : ffff80001027a434: ldrh w4, [x0] 0.00 : ffff80001027a438: mov w1, #0x408 // #1032 0.00 : ffff80001027a43c: and w1, w1, w4 0.00 : ffff80001027a440: cmp w1, #0x400 0.00 : ffff80001027a444: b.ne ffff80001027a3e0 // b.any : rw_verify_area(): : retval = locks_mandatory_area(inode, file, pos, pos + count - 1, 0.00 : ffff80001027a448: cmp w20, #0x0 0.00 : ffff80001027a44c: sub x3, x3, #0x1 0.00 : ffff80001027a450: cset w4, ne // ne = any 0.00 : ffff80001027a454: add x3, x3, x2 0.00 : ffff80001027a458: mov x1, x19 0.00 : ffff80001027a45c: bl ffff8000102ee050 : if (retval < 0) 0.00 : ffff80001027a460: tbz w0, #31, ffff80001027a3e0 0.00 : ffff80001027a464: b ffff80001027a3f8 : unsigned_offsets(): : return file->f_mode & FMODE_UNSIGNED_OFFSET; 0.00 : ffff80001027a468: ldr w0, [x1, #68] : rw_verify_area(): : if (!unsigned_offsets(file)) 0.00 : ffff80001027a46c: tbnz w0, #13, ffff80001027a3d4 : return retval; 0.00 : ffff80001027a470: mov w0, #0xffffffea // #-22 0.00 : ffff80001027a474: b ffff80001027a3f8 Percent | Source code & Disassembly of vmlinux for cycles (35 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010125598 : : update_group_capacity(): : sdg->sgc->min_capacity = capacity; : sdg->sgc->max_capacity = capacity; : } : : void update_group_capacity(struct sched_domain *sd, int cpu) : { 0.00 : ffff800010125598: stp x29, x30, [sp, #-80]! 0.00 : ffff80001012559c: mov x29, sp 0.00 : ffff8000101255a0: stp x19, x20, [sp, #16] 0.00 : ffff8000101255a4: mov x19, x0 0.00 : ffff8000101255a8: str x21, [sp, #32] 0.00 : ffff8000101255ac: sxtw x21, w1 0.00 : ffff8000101255b0: str x25, [sp, #64] : msecs_to_jiffies(): : if (__builtin_constant_p(m)) { : if ((int)m < 0) : return MAX_JIFFY_OFFSET; : return _msecs_to_jiffies(m); : } else { : return __msecs_to_jiffies(m); 0.00 : ffff8000101255b4: ldr w0, [x0, #72] : update_group_capacity(): : struct sched_domain *child = sd->child; : struct sched_group *group, *sdg = sd->groups; 0.00 : ffff8000101255b8: ldp x20, x25, [x19, #8] : msecs_to_jiffies(): 0.00 : ffff8000101255bc: bl ffff800010167188 <__msecs_to_jiffies> : update_group_capacity(): : unsigned long capacity, min_capacity, max_capacity; : unsigned long interval; : : interval = msecs_to_jiffies(sd->balance_interval); : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff8000101255c0: cmp x0, #0x0 0.00 : ffff8000101255c4: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : sdg->sgc->next_update = jiffies + interval; 0.00 : ffff8000101255c8: adrp x4, ffff800011907000 : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff8000101255cc: mov x3, #0x1 // #1 : sdg->sgc->next_update = jiffies + interval; 0.00 : ffff8000101255d0: ldr x2, [x25, #16] : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff8000101255d4: ldr x1, [x1, #584] 0.00 : ffff8000101255d8: csel x0, x0, x3, ne // ne = any : sdg->sgc->next_update = jiffies + interval; 0.00 : ffff8000101255dc: ldr x4, [x4, #2432] : interval = clamp(interval, 1UL, max_load_balance_interval); 0.00 : ffff8000101255e0: cmp x0, x1 0.00 : ffff8000101255e4: csel x0, x0, x1, ls // ls = plast : sdg->sgc->next_update = jiffies + interval; 0.00 : ffff8000101255e8: add x0, x0, x4 0.00 : ffff8000101255ec: str x0, [x2, #32] : : if (!child) { 0.00 : ffff8000101255f0: cbz x20, ffff8000101256f0 : : capacity = 0; : min_capacity = ULONG_MAX; : max_capacity = 0; : : if (child->flags & SD_OVERLAP) { 0.00 : ffff8000101255f4: ldr w0, [x20, #56] 0.00 : ffff8000101255f8: tbz w0, #13, ffff8000101256a8 0.00 : ffff8000101255fc: stp x22, x23, [x29, #40] : min_capacity = ULONG_MAX; 0.00 : ffff800010125600: mov x19, #0xffffffffffffffff // #-1 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010125604: adrp x22, ffff800011528000 0.00 : ffff800010125608: adrp x23, ffff800011909000 0.00 : ffff80001012560c: str x24, [x29, #56] : update_group_capacity(): : max_capacity = 0; 0.00 : ffff800010125610: mov x20, #0x0 // #0 0.00 : ffff800010125614: str x26, [x29, #72] : capacity = 0; 0.00 : ffff800010125618: mov x21, #0x0 // #0 0.00 : ffff80001012561c: adrp x26, ffff80001190a000 <__per_cpu_offset+0x6d8> : /* : * SD_OVERLAP domains cannot assume that child groups : * span the current group. : */ : : for_each_cpu(cpu, sched_group_span(sdg)) { 0.00 : ffff800010125620: mov w0, w19 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010125624: add x22, x22, #0xe80 0.00 : ffff800010125628: add x23, x23, #0x928 0.00 : ffff80001012562c: add x24, x25, #0x20 : update_group_capacity(): : for_each_cpu(cpu, sched_group_span(sdg)) { 0.00 : ffff800010125630: add x26, x26, #0x2f0 0.00 : ffff800010125634: b ffff800010125658 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 2.21 : ffff800010125638: ldr x2, [x23, w0, sxtw #3] 0.00 : ffff80001012563c: add x1, x2, x1 5.00 : ffff800010125640: ldr x1, [x1, #2480] : update_group_capacity(): : unsigned long cpu_cap = capacity_of(cpu); : : capacity += cpu_cap; : min_capacity = min(cpu_cap, min_capacity); 0.00 : ffff800010125644: cmp x19, x1 : capacity += cpu_cap; 0.00 : ffff800010125648: add x21, x21, x1 : min_capacity = min(cpu_cap, min_capacity); 0.00 : ffff80001012564c: csel x19, x19, x1, ls // ls = plast : max_capacity = max(cpu_cap, max_capacity); 14.76 : ffff800010125650: cmp x20, x1 0.00 : ffff800010125654: csel x20, x20, x1, cs // cs = hs, nlast : for_each_cpu(cpu, sched_group_span(sdg)) { 0.00 : ffff800010125658: mov x1, x24 0.00 : ffff80001012565c: bl ffff800010cb92d8 0.00 : ffff800010125660: ldr w2, [x26] : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010125664: mov x1, x22 : update_group_capacity(): : for_each_cpu(cpu, sched_group_span(sdg)) { 0.00 : ffff800010125668: cmp w0, w2 0.00 : ffff80001012566c: b.cc ffff800010125638 // b.lo, b.ul, b.last 0.00 : ffff800010125670: ldp x22, x23, [x29, #40] 0.00 : ffff800010125674: ldr x24, [x29, #56] 0.00 : ffff800010125678: ldr x26, [x29, #72] : max_capacity = max(sgc->max_capacity, max_capacity); : group = group->next; : } while (group != child->groups); : } : : sdg->sgc->capacity = capacity; 0.00 : ffff80001012567c: ldr x0, [x25, #16] 0.00 : ffff800010125680: str x21, [x0, #8] : sdg->sgc->min_capacity = min_capacity; 0.00 : ffff800010125684: ldr x0, [x25, #16] 0.00 : ffff800010125688: str x19, [x0, #16] : sdg->sgc->max_capacity = max_capacity; 0.00 : ffff80001012568c: ldr x0, [x25, #16] 0.00 : ffff800010125690: str x20, [x0, #24] : } 0.00 : ffff800010125694: ldp x19, x20, [sp, #16] 0.00 : ffff800010125698: ldr x21, [sp, #32] 0.00 : ffff80001012569c: ldr x25, [sp, #64] 0.00 : ffff8000101256a0: ldp x29, x30, [sp], #80 0.00 : ffff8000101256a4: ret : group = child->groups; 0.00 : ffff8000101256a8: ldr x4, [x20, #16] : min_capacity = ULONG_MAX; 0.00 : ffff8000101256ac: mov x19, #0xffffffffffffffff // #-1 : capacity = 0; 0.00 : ffff8000101256b0: mov x21, #0x0 // #0 : max_capacity = 0; 0.00 : ffff8000101256b4: mov x20, #0x0 // #0 : group = child->groups; 0.00 : ffff8000101256b8: mov x0, x4 0.00 : ffff8000101256bc: nop : struct sched_group_capacity *sgc = group->sgc; 0.00 : ffff8000101256c0: ldr x1, [x0, #16] : group = group->next; 5.15 : ffff8000101256c4: ldr x0, [x0] : max_capacity = max(sgc->max_capacity, max_capacity); 0.00 : ffff8000101256c8: ldp x3, x2, [x1, #16] : capacity += sgc->capacity; 24.42 : ffff8000101256cc: ldr x1, [x1, #8] : min_capacity = min(sgc->min_capacity, min_capacity); 0.00 : ffff8000101256d0: cmp x19, x3 0.00 : ffff8000101256d4: csel x19, x19, x3, ls // ls = plast : max_capacity = max(sgc->max_capacity, max_capacity); 0.00 : ffff8000101256d8: cmp x20, x2 6.27 : ffff8000101256dc: csel x20, x20, x2, cs // cs = hs, nlast : capacity += sgc->capacity; 0.00 : ffff8000101256e0: add x21, x21, x1 : } while (group != child->groups); 0.00 : ffff8000101256e4: cmp x4, x0 0.00 : ffff8000101256e8: b.ne ffff8000101256c0 // b.any 0.00 : ffff8000101256ec: b ffff80001012567c : scale_rt_capacity(): : struct rq *rq = cpu_rq(cpu); 0.00 : ffff8000101256f0: adrp x6, ffff800011909000 0.00 : ffff8000101256f4: add x2, x6, #0x928 0.00 : ffff8000101256f8: adrp x4, ffff800011528000 0.00 : ffff8000101256fc: add x1, x4, #0xe80 0.00 : ffff800010125700: ldr x2, [x2, x21, lsl #3] : topology_get_cpu_scale(): : : struct sched_domain; : static inline : unsigned long topology_get_cpu_scale(int cpu) : { : return per_cpu(cpu_scale, cpu); 0.00 : ffff800010125704: adrp x0, ffff800011527000 0.00 : ffff800010125708: add x0, x0, #0xa00 0.00 : ffff80001012570c: ldr x8, [x19, #16] : scale_rt_capacity(): 0.00 : ffff800010125710: add x1, x1, x2 : topology_get_cpu_scale(): 0.00 : ffff800010125714: ldr x5, [x2, x0] : scale_rt_capacity(): : irq = cpu_util_irq(rq); 36.32 : ffff800010125718: ldr x7, [x1, #2800] : if (unlikely(irq >= max)) 0.00 : ffff80001012571c: cmp x5, x7 0.00 : ffff800010125720: b.ls ffff800010125794 // b.plast : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010125724: ldr x2, [x1, #2672] 0.00 : ffff800010125728: ldr x0, [x1, #2736] : scale_rt_capacity(): : used += READ_ONCE(rq->avg_dl.util_avg); 0.00 : ffff80001012572c: add x2, x2, x0 : if (unlikely(used >= max)) 0.00 : ffff800010125730: cmp x5, x2 0.00 : ffff800010125734: b.ls ffff800010125794 // b.plast : free = max - used; 2.71 : ffff800010125738: sub x0, x5, x2 : scale_irq_capacity(): : } : : static inline : unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max) : { : util *= (max - irq); 0.00 : ffff80001012573c: sub x7, x5, x7 : update_cpu_capacity(): : cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu); 0.00 : ffff800010125740: str x5, [x1, #2488] : scale_irq_capacity(): 0.00 : ffff800010125744: mul x0, x0, x7 : util /= max; 0.00 : ffff800010125748: udiv x0, x0, x5 : update_cpu_capacity(): : capacity = 1; 0.00 : ffff80001012574c: cmp x0, #0x0 0.00 : ffff800010125750: csel x0, x0, x3, ne // ne = any : cpu_rq(cpu)->cpu_capacity = capacity; 0.00 : ffff800010125754: add x6, x6, #0x928 0.00 : ffff800010125758: add x1, x4, #0xe80 0.00 : ffff80001012575c: ldr x2, [x6, x21, lsl #3] 0.00 : ffff800010125760: add x1, x2, x1 0.00 : ffff800010125764: str x0, [x1, #2480] : sdg->sgc->capacity = capacity; 0.00 : ffff800010125768: ldr x1, [x8, #16] 0.00 : ffff80001012576c: str x0, [x1, #8] : sdg->sgc->min_capacity = capacity; 0.00 : ffff800010125770: ldr x1, [x8, #16] 0.00 : ffff800010125774: str x0, [x1, #16] : sdg->sgc->max_capacity = capacity; 0.00 : ffff800010125778: ldr x1, [x8, #16] 0.00 : ffff80001012577c: str x0, [x1, #24] : update_group_capacity(): : } 0.00 : ffff800010125780: ldp x19, x20, [sp, #16] 3.15 : ffff800010125784: ldr x21, [sp, #32] 0.00 : ffff800010125788: ldr x25, [sp, #64] 0.00 : ffff80001012578c: ldp x29, x30, [sp], #80 0.00 : ffff800010125790: ret : scale_rt_capacity(): : return 1; 0.00 : ffff800010125794: mov x0, x3 : update_cpu_capacity(): : cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu); 0.00 : ffff800010125798: str x5, [x1, #2488] 0.00 : ffff80001012579c: b ffff800010125754 Percent | Source code & Disassembly of vmlinux for cycles (70 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010712828 <__iommu_map>: : __iommu_map(): : return pgsize; : } : : int __iommu_map(struct iommu_domain *domain, unsigned long iova, : phys_addr_t paddr, size_t size, int prot, gfp_t gfp) : { 1.46 : ffff800010712828: stp x29, x30, [sp, #-112]! 0.00 : ffff80001071282c: mov x29, sp 0.00 : ffff800010712830: stp x21, x22, [sp, #32] 0.00 : ffff800010712834: mov x21, x0 0.00 : ffff800010712838: str x28, [sp, #88] 4.36 : ffff80001071283c: str x2, [x29, #104] : const struct iommu_ops *ops = domain->ops; 0.00 : ffff800010712840: ldr x22, [x0, #8] : unsigned int min_pagesz; : size_t orig_size = size; : phys_addr_t orig_paddr = paddr; : int ret = 0; : : if (unlikely(ops->map == NULL || 7.23 : ffff800010712844: ldr x0, [x22, #40] 0.00 : ffff800010712848: cbz x0, ffff800010712990 <__iommu_map+0x168> 48.07 : ffff80001071284c: ldr x0, [x21, #16] 0.00 : ffff800010712850: cbz x0, ffff800010712990 <__iommu_map+0x168> : domain->pgsize_bitmap == 0UL)) : return -ENODEV; : : if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 4.19 : ffff800010712854: ldr w6, [x21] 0.00 : ffff800010712858: tbz w6, #0, ffff8000107129a8 <__iommu_map+0x180> : phys_addr_t orig_paddr = paddr; 0.00 : ffff80001071285c: mov x6, x2 : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 0.00 : ffff800010712860: rbit x2, x0 0.00 : ffff800010712864: clz x2, x2 0.00 : ffff800010712868: stp x23, x24, [x29, #48] 0.00 : ffff80001071286c: stp x25, x26, [x29, #64] 0.00 : ffff800010712870: mov w24, w5 0.00 : ffff800010712874: mov w23, w4 : __iommu_map(): : /* : * both the virtual address and the physical one, as well as : * the size of the mapping, must be aligned (at least) to the : * size of the smallest page supported by the hardware : */ : if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 0.00 : ffff800010712878: orr x5, x3, x6 : min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 1.45 : ffff80001071287c: mov w4, #0x1 // #1 : if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 0.00 : ffff800010712880: orr x5, x5, x1 : min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 0.00 : ffff800010712884: lsl w2, w4, w2 0.00 : ffff800010712888: mov x4, x2 : if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 0.00 : ffff80001071288c: sub x2, x2, #0x1 0.00 : ffff800010712890: mov x26, x1 0.00 : ffff800010712894: mov x25, x3 0.00 : ffff800010712898: tst x5, x2 0.00 : ffff80001071289c: b.ne ffff8000107129b0 <__iommu_map+0x188> // b.any 0.00 : ffff8000107128a0: str x20, [x29, #24] : return -EINVAL; : } : : pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); : : while (size) { 0.00 : ffff8000107128a4: mov x20, x1 2.86 : ffff8000107128a8: str x27, [x29, #80] 0.00 : ffff8000107128ac: mov x27, x3 0.00 : ffff8000107128b0: cbz x3, ffff800010712958 <__iommu_map+0x130> 0.00 : ffff8000107128b4: str x19, [x29, #16] 0.00 : ffff8000107128b8: b ffff8000107128d4 <__iommu_map+0xac> : : if (ret) : break; : : iova += pgsize; : paddr += pgsize; 2.93 : ffff8000107128bc: ldr x6, [x29, #104] : while (size) { 0.00 : ffff8000107128c0: subs x27, x27, x19 : paddr += pgsize; 0.00 : ffff8000107128c4: add x6, x19, x6 7.26 : ffff8000107128c8: str x6, [x29, #104] : while (size) { 0.00 : ffff8000107128cc: b.eq ffff800010712954 <__iommu_map+0x12c> // b.none 0.00 : ffff8000107128d0: ldr x0, [x21, #16] : size_t pgsize = iommu_pgsize(domain, iova | paddr, size); 0.00 : ffff8000107128d4: orr x1, x6, x20 0.00 : ffff8000107128d8: mov x2, x27 0.00 : ffff8000107128dc: bl ffff800010711750 0.00 : ffff8000107128e0: mov x19, x0 : ret = ops->map(domain, iova, paddr, pgsize, prot, gfp); 0.00 : ffff8000107128e4: ldr x7, [x22, #40] 0.00 : ffff8000107128e8: mov x1, x20 0.00 : ffff8000107128ec: mov x0, x21 0.00 : ffff8000107128f0: mov x2, x6 2.92 : ffff8000107128f4: mov w5, w24 0.00 : ffff8000107128f8: mov w4, w23 0.00 : ffff8000107128fc: mov x3, x19 : iova += pgsize; 0.00 : ffff800010712900: add x20, x20, x19 : ret = ops->map(domain, iova, paddr, pgsize, prot, gfp); 0.00 : ffff800010712904: blr x7 4.33 : ffff800010712908: mov w28, w0 : if (ret) 0.00 : ffff80001071290c: cbz w0, ffff8000107128bc <__iommu_map+0x94> : size -= pgsize; : } : : if (ops->iotlb_sync_map) 0.00 : ffff800010712910: ldr x1, [x22, #64] 0.00 : ffff800010712914: cbz x1, ffff800010712920 <__iommu_map+0xf8> : ops->iotlb_sync_map(domain); 0.00 : ffff800010712918: mov x0, x21 0.00 : ffff80001071291c: blr x1 : : /* unroll mapping in case something went wrong */ : if (ret) : iommu_unmap(domain, orig_iova, orig_size - size); 0.00 : ffff800010712920: sub x2, x25, x27 0.00 : ffff800010712924: mov x1, x26 0.00 : ffff800010712928: mov x0, x21 0.00 : ffff80001071292c: bl ffff8000107118c0 0.00 : ffff800010712930: ldp x19, x20, [x29, #16] 0.00 : ffff800010712934: ldp x23, x24, [x29, #48] 0.00 : ffff800010712938: ldp x25, x26, [x29, #64] 0.00 : ffff80001071293c: ldr x27, [x29, #80] : else : trace_map(orig_iova, orig_paddr, orig_size); : : return ret; : } 0.00 : ffff800010712940: mov w0, w28 0.00 : ffff800010712944: ldr x28, [sp, #88] 0.00 : ffff800010712948: ldp x21, x22, [sp, #32] 2.89 : ffff80001071294c: ldp x29, x30, [sp], #112 0.00 : ffff800010712950: ret 1.45 : ffff800010712954: ldr x19, [x29, #16] : if (ops->iotlb_sync_map) 1.44 : ffff800010712958: ldr x1, [x22, #64] 0.00 : ffff80001071295c: cbz x1, ffff8000107129d0 <__iommu_map+0x1a8> : ops->iotlb_sync_map(domain); 0.00 : ffff800010712960: mov x0, x21 0.00 : ffff800010712964: mov w28, #0x0 // #0 0.00 : ffff800010712968: blr x1 0.00 : ffff80001071296c: ldr x20, [x29, #24] 0.00 : ffff800010712970: ldp x23, x24, [x29, #48] : } 0.00 : ffff800010712974: mov w0, w28 0.00 : ffff800010712978: ldp x25, x26, [x29, #64] 0.00 : ffff80001071297c: ldr x27, [x29, #80] 0.00 : ffff800010712980: ldp x21, x22, [sp, #32] 0.00 : ffff800010712984: ldr x28, [sp, #88] 0.00 : ffff800010712988: ldp x29, x30, [sp], #112 0.00 : ffff80001071298c: ret : return -ENODEV; 0.00 : ffff800010712990: mov w28, #0xffffffed // #-19 : } 0.00 : ffff800010712994: mov w0, w28 0.00 : ffff800010712998: ldp x21, x22, [sp, #32] 0.00 : ffff80001071299c: ldr x28, [sp, #88] 0.00 : ffff8000107129a0: ldp x29, x30, [sp], #112 0.00 : ffff8000107129a4: ret : return -EINVAL; 0.00 : ffff8000107129a8: mov w28, #0xffffffea // #-22 0.00 : ffff8000107129ac: b ffff800010712940 <__iommu_map+0x118> : pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", 0.00 : ffff8000107129b0: adrp x0, ffff800011269000 0.00 : ffff8000107129b4: add x2, x29, #0x68 0.00 : ffff8000107129b8: add x0, x0, #0x8e8 : return -EINVAL; 0.00 : ffff8000107129bc: mov w28, #0xffffffea // #-22 : pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", 0.00 : ffff8000107129c0: bl ffff80001014a85c : return -EINVAL; 0.00 : ffff8000107129c4: ldp x23, x24, [x29, #48] 0.00 : ffff8000107129c8: ldp x25, x26, [x29, #64] 0.00 : ffff8000107129cc: b ffff800010712940 <__iommu_map+0x118> : if (ops->iotlb_sync_map) 1.45 : ffff8000107129d0: mov w28, #0x0 // #0 0.00 : ffff8000107129d4: ldr x20, [x29, #24] 4.25 : ffff8000107129d8: ldp x23, x24, [x29, #48] 1.46 : ffff8000107129dc: ldp x25, x26, [x29, #64] 0.00 : ffff8000107129e0: ldr x27, [x29, #80] 0.00 : ffff8000107129e4: b ffff800010712940 <__iommu_map+0x118> Percent | Source code & Disassembly of vmlinux for cycles (62 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001046f0e8 : : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001046f0e8: ldr x6, [x0, #64] : disk_map_sector_rcu(): : * : * RETURNS: : * Found partition on success, part0 is returned if no partition matches : */ : struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector) : { 0.00 : ffff80001046f0ec: mov x7, x0 : __read_once_size(): 25.79 : ffff80001046f0f0: ldr x2, [x6, #24] : disk_map_sector_rcu(): : int i; : : ptbl = rcu_dereference(disk->part_tbl); : : part = rcu_dereference(ptbl->last_lookup); : if (part && sector_in_part(part, sector)) 0.00 : ffff80001046f0f4: cbz x2, ffff80001046f108 : part = rcu_dereference(ptbl->last_lookup); 41.81 : ffff80001046f0f8: mov x0, x2 : sector_in_part(): : return part->start_sect <= sector && 1.62 : ffff80001046f0fc: ldr x2, [x2] 3.24 : ffff80001046f100: cmp x1, x2 0.00 : ffff80001046f104: b.cs ffff80001046f164 // b.hs, b.nlast : disk_map_sector_rcu(): : return part; : : for (i = 1; i < ptbl->len; i++) { 0.00 : ffff80001046f108: ldr w5, [x6, #16] 0.00 : ffff80001046f10c: cmp w5, #0x1 0.00 : ffff80001046f110: b.le ffff80001046f15c 0.00 : ffff80001046f114: sub w3, w5, #0x2 0.00 : ffff80001046f118: add x5, x6, #0x30 0.00 : ffff80001046f11c: add x2, x6, #0x28 0.00 : ffff80001046f120: add x5, x5, w3, uxtw #3 0.00 : ffff80001046f124: nop : __read_once_size(): 0.00 : ffff80001046f128: ldr x3, [x2] 0.00 : ffff80001046f12c: add x2, x2, #0x8 : disk_map_sector_rcu(): : part = rcu_dereference(ptbl->part[i]); 0.00 : ffff80001046f130: mov x0, x3 : : if (part && sector_in_part(part, sector)) { 0.00 : ffff80001046f134: cbz x3, ffff80001046f154 : sector_in_part(): : return part->start_sect <= sector && 0.00 : ffff80001046f138: ldr x4, [x3] 0.00 : ffff80001046f13c: cmp x1, x4 0.00 : ffff80001046f140: b.cc ffff80001046f154 // b.lo, b.ul, b.last : sector < part->start_sect + part_nr_sects_read(part); 0.00 : ffff80001046f144: ldr x3, [x3, #8] 0.00 : ffff80001046f148: add x4, x4, x3 : return part->start_sect <= sector && 0.00 : ffff80001046f14c: cmp x1, x4 0.00 : ffff80001046f150: b.cc ffff80001046f178 // b.lo, b.ul, b.last : disk_map_sector_rcu(): : for (i = 1; i < ptbl->len; i++) { 0.00 : ffff80001046f154: cmp x2, x5 0.00 : ffff80001046f158: b.ne ffff80001046f128 // b.any : rcu_assign_pointer(ptbl->last_lookup, part); : return part; : } : } : return &disk->part0; 0.00 : ffff80001046f15c: add x0, x7, #0x48 : } 0.00 : ffff80001046f160: ret : sector_in_part(): : sector < part->start_sect + part_nr_sects_read(part); 25.92 : ffff80001046f164: ldr x3, [x0, #8] 0.00 : ffff80001046f168: add x2, x2, x3 : return part->start_sect <= sector && 0.00 : ffff80001046f16c: cmp x1, x2 0.00 : ffff80001046f170: b.cs ffff80001046f108 // b.hs, b.nlast : disk_map_sector_rcu(): : } 1.62 : ffff80001046f174: ret : rcu_assign_pointer(ptbl->last_lookup, part); 0.00 : ffff80001046f178: add x1, x6, #0x18 0.00 : ffff80001046f17c: stlr x0, [x1] : } 0.00 : ffff80001046f180: ret Percent | Source code & Disassembly of vmlinux for cycles (67 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd4428 : : mutex_unlock(): : * of a not locked mutex is not allowed. : * : * This function is similar to (but not equivalent to) up(). : */ : void __sched mutex_unlock(struct mutex *lock) : { 0.00 : ffff800010cd4428: stp x29, x30, [sp, #-16]! 0.00 : ffff800010cd442c: mov x3, x0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 3.03 : ffff800010cd4430: mrs x4, sp_el0 : mutex_unlock(): 0.00 : ffff800010cd4434: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010cd4438: b ffff800010cd4458 1.52 : ffff800010cd443c: b ffff800010cd4458 : __lse__cmpxchg_case_rel_64(): : __CMPXCHG_CASE(w, , acq_, 32, a, "memory") : __CMPXCHG_CASE(x, , acq_, 64, a, "memory") : __CMPXCHG_CASE(w, b, rel_, 8, l, "memory") : __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") 0.00 : ffff800010cd4440: mov x2, #0x0 // #0 0.00 : ffff800010cd4444: mov x1, x4 0.00 : ffff800010cd4448: mov x5, x1 0.00 : ffff800010cd444c: casl x5, x2, [x0] 93.96 : ffff800010cd4450: mov x0, x5 0.00 : ffff800010cd4454: b ffff800010cd4460 : __ll_sc__cmpxchg_case_rel_64(): : __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) : __CMPXCHG_CASE( , , acq_, 64, , a, , "memory", L) : __CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory", K) : __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) 0.00 : ffff800010cd4458: mov x1, #0x0 // #0 0.00 : ffff800010cd445c: b ffff800010cd56c8 : __mutex_unlock_fast(): : if (atomic_long_cmpxchg_release(&lock->owner, curr, 0UL) == curr) 0.00 : ffff800010cd4460: cmp x4, x0 0.00 : ffff800010cd4464: b.eq ffff800010cd4470 // b.none 0.00 : ffff800010cd4468: mov x0, x3 : mutex_unlock(): : #ifndef CONFIG_DEBUG_LOCK_ALLOC : if (__mutex_unlock_fast(lock)) : return; : #endif : __mutex_unlock_slowpath(lock, _RET_IP_); 0.00 : ffff800010cd446c: bl ffff800010cd42a8 <__mutex_unlock_slowpath.isra.19> : } 1.50 : ffff800010cd4470: ldp x29, x30, [sp], #16 0.00 : ffff800010cd4474: ret Percent | Source code & Disassembly of vmlinux for cycles (98 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000107d0750 : : nvme_setup_cmd(): : } : EXPORT_SYMBOL_GPL(nvme_cleanup_cmd); : : blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req, : struct nvme_command *cmd) : { 0.00 : ffff8000107d0750: stp x29, x30, [sp, #-32]! 0.00 : ffff8000107d0754: mov x29, sp 12.33 : ffff8000107d0758: stp x19, x20, [sp, #16] 0.00 : ffff8000107d075c: mov x20, x1 1.04 : ffff8000107d0760: mov x19, x2 : nvme_clear_nvme_request(): : if (!(req->rq_flags & RQF_DONTPREP)) { 0.00 : ffff8000107d0764: ldr w1, [x1, #28] 0.00 : ffff8000107d0768: tbnz w1, #7, ffff8000107d0778 : req->rq_flags |= RQF_DONTPREP; 1.03 : ffff8000107d076c: orr w1, w1, #0x80 0.00 : ffff8000107d0770: str w1, [x20, #28] : nvme_req(req)->retries = 0; 0.00 : ffff8000107d0774: strh wzr, [x20, #296] : nvme_setup_cmd(): : blk_status_t ret = BLK_STS_OK; : : nvme_clear_nvme_request(req); : : memset(cmd, 0, sizeof(*cmd)); 1.06 : ffff8000107d0778: stp xzr, xzr, [x19] 1.03 : ffff8000107d077c: stp xzr, xzr, [x19, #16] 1.03 : ffff8000107d0780: stp xzr, xzr, [x19, #32] 0.00 : ffff8000107d0784: stp xzr, xzr, [x19, #48] : switch (req_op(req)) { 2.05 : ffff8000107d0788: ldr w2, [x20, #24] 0.00 : ffff8000107d078c: and w1, w2, #0xff 0.00 : ffff8000107d0790: cmp w1, #0x3 0.00 : ffff8000107d0794: b.eq ffff8000107d0814 // b.none 4.12 : ffff8000107d0798: b.hi ffff8000107d07d0 // b.pmore 1.98 : ffff8000107d079c: cmp w1, #0x1 1.02 : ffff8000107d07a0: b.ls ffff8000107d0894 // b.plast : nvme_setup_flush(): : cmnd->common.opcode = nvme_cmd_flush; 0.00 : ffff8000107d07a4: strb wzr, [x19] : nvme_setup_cmd(): : blk_status_t ret = BLK_STS_OK; 0.00 : ffff8000107d07a8: mov w1, #0x0 // #0 : nvme_setup_flush(): : cmnd->common.nsid = cpu_to_le32(ns->head->ns_id); 0.00 : ffff8000107d07ac: ldr x0, [x0, #72] 0.00 : ffff8000107d07b0: ldr w0, [x0, #1968] 0.00 : ffff8000107d07b4: str w0, [x19, #4] : nvme_setup_cmd(): : default: : WARN_ON_ONCE(1); : return BLK_STS_IOERR; : } : : cmd->common.command_id = req->tag; 2.03 : ffff8000107d07b8: ldr w0, [x20, #32] 0.00 : ffff8000107d07bc: strh w0, [x19, #2] : trace_nvme_setup_cmd(req, cmd); : return ret; : } 0.00 : ffff8000107d07c0: mov w0, w1 0.00 : ffff8000107d07c4: ldp x19, x20, [sp, #16] 4.12 : ffff8000107d07c8: ldp x29, x30, [sp], #32 0.00 : ffff8000107d07cc: ret : switch (req_op(req)) { 0.00 : ffff8000107d07d0: cmp w1, #0x9 0.00 : ffff8000107d07d4: b.eq ffff8000107d0840 // b.none 0.00 : ffff8000107d07d8: b.cc ffff8000107d0828 // b.lo, b.ul, b.last 0.00 : ffff8000107d07dc: sub w1, w1, #0x22 0.00 : ffff8000107d07e0: cmp w1, #0x1 0.00 : ffff8000107d07e4: b.hi ffff8000107d0828 // b.pmore : memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd)); 0.00 : ffff8000107d07e8: ldr x0, [x20, #280] : blk_status_t ret = BLK_STS_OK; 0.00 : ffff8000107d07ec: mov w1, #0x0 // #0 : memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd)); 0.00 : ffff8000107d07f0: ldp x2, x3, [x0] 0.00 : ffff8000107d07f4: stp x2, x3, [x19] 0.00 : ffff8000107d07f8: ldp x2, x3, [x0, #16] 0.00 : ffff8000107d07fc: stp x2, x3, [x19, #16] 0.00 : ffff8000107d0800: ldp x2, x3, [x0, #32] 0.00 : ffff8000107d0804: stp x2, x3, [x19, #32] 0.00 : ffff8000107d0808: ldp x2, x3, [x0, #48] 0.00 : ffff8000107d080c: stp x2, x3, [x19, #48] : break; 0.00 : ffff8000107d0810: b ffff8000107d07b8 : ret = nvme_setup_discard(ns, req, cmd); 0.00 : ffff8000107d0814: mov x1, x20 0.00 : ffff8000107d0818: mov x2, x19 0.00 : ffff8000107d081c: bl ffff8000107d0550 0.00 : ffff8000107d0820: and w1, w0, #0xff : break; 0.00 : ffff8000107d0824: b ffff8000107d07b8 : WARN_ON_ONCE(1); 0.00 : ffff8000107d0828: brk #0x800 : return BLK_STS_IOERR; 0.00 : ffff8000107d082c: mov w1, #0xa // #10 : } 0.00 : ffff8000107d0830: mov w0, w1 0.00 : ffff8000107d0834: ldp x19, x20, [sp, #16] 0.00 : ffff8000107d0838: ldp x29, x30, [sp], #32 0.00 : ffff8000107d083c: ret : nvme_setup_write_zeroes(): : if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES) 0.00 : ffff8000107d0840: ldr x1, [x0, #16] 0.00 : ffff8000107d0844: ldr x1, [x1, #1288] 0.00 : ffff8000107d0848: tbnz w1, #2, ffff8000107d0814 : cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes; 0.00 : ffff8000107d084c: mov w1, #0x8 // #8 0.00 : ffff8000107d0850: strb w1, [x19] : return BLK_STS_OK; 0.00 : ffff8000107d0854: mov w1, #0x0 // #0 : cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id); 0.00 : ffff8000107d0858: ldr x2, [x0, #72] 0.00 : ffff8000107d085c: ldr w2, [x2, #1968] 0.00 : ffff8000107d0860: str w2, [x19, #4] : nvme_sect_to_lba(): : /* : * Convert a 512B sector number to a device logical block number. : */ : static inline u64 nvme_sect_to_lba(struct nvme_ns *ns, sector_t sector) : { : return sector >> (ns->lba_shift - SECTOR_SHIFT); 0.00 : ffff8000107d0864: ldr w3, [x0, #80] 0.00 : ffff8000107d0868: ldr x2, [x20, #48] 0.00 : ffff8000107d086c: sub w3, w3, #0x9 0.00 : ffff8000107d0870: lsr x2, x2, x3 : nvme_setup_write_zeroes(): : cmnd->write_zeroes.slba = 0.00 : ffff8000107d0874: str x2, [x19, #40] : cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 0.00 : ffff8000107d0878: ldr w2, [x0, #80] 0.00 : ffff8000107d087c: ldr w0, [x20, #40] : cmnd->write_zeroes.control = 0; 0.00 : ffff8000107d0880: strh wzr, [x19, #50] : cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 0.00 : ffff8000107d0884: lsr w0, w0, w2 0.00 : ffff8000107d0888: sub w0, w0, #0x1 : cmnd->write_zeroes.length = 0.00 : ffff8000107d088c: strh w0, [x19, #48] 0.00 : ffff8000107d0890: b ffff8000107d07b8 : nvme_setup_rw(): : u16 control = 0; 1.03 : ffff8000107d0894: ubfx x1, x2, #17, #1 : if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 0.00 : ffff8000107d0898: and w4, w2, #0xfff00 0.00 : ffff8000107d089c: and w4, w4, #0xfff801ff : u32 dsmgmt = 0; 0.00 : ffff8000107d08a0: mov w3, #0x7 // #7 : u16 control = 0; 2.11 : ffff8000107d08a4: lsl w1, w1, #14 : control |= NVME_RW_LR; 0.00 : ffff8000107d08a8: cmp w4, #0x0 0.00 : ffff8000107d08ac: orr w4, w1, #0xffff8000 : struct nvme_ctrl *ctrl = ns->ctrl; 0.00 : ffff8000107d08b0: ldr x5, [x0, #16] : control |= NVME_RW_LR; 0.00 : ffff8000107d08b4: and w4, w4, #0xffff 0.00 : ffff8000107d08b8: csel w1, w4, w1, ne // ne = any : u32 dsmgmt = 0; 0.00 : ffff8000107d08bc: tst x2, #0x80000 29.40 : ffff8000107d08c0: csel w3, w3, wzr, ne // ne = any : cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read); 0.00 : ffff8000107d08c4: tst x2, #0x1 0.00 : ffff8000107d08c8: cset w2, eq // eq = none 0.00 : ffff8000107d08cc: add w2, w2, #0x1 0.00 : ffff8000107d08d0: strb w2, [x19] : cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id); 0.00 : ffff8000107d08d4: ldr x2, [x0, #72] 5.08 : ffff8000107d08d8: ldr w2, [x2, #1968] 26.41 : ffff8000107d08dc: str w2, [x19, #4] : nvme_sect_to_lba(): 0.00 : ffff8000107d08e0: ldr w4, [x0, #80] 0.00 : ffff8000107d08e4: ldr x2, [x20, #48] 0.00 : ffff8000107d08e8: sub w4, w4, #0x9 0.00 : ffff8000107d08ec: lsr x2, x2, x4 : nvme_setup_rw(): : cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); 0.00 : ffff8000107d08f0: str x2, [x19, #40] : cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 2.09 : ffff8000107d08f4: ldr w4, [x0, #80] 0.00 : ffff8000107d08f8: ldr w2, [x20, #40] 0.00 : ffff8000107d08fc: lsr w2, w2, w4 0.00 : ffff8000107d0900: sub w2, w2, #0x1 0.00 : ffff8000107d0904: strh w2, [x19, #48] : if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams) 1.03 : ffff8000107d0908: ldrb w2, [x20, #24] 0.00 : ffff8000107d090c: cmp w2, #0x1 0.00 : ffff8000107d0910: b.eq ffff8000107d0974 // b.none : if (ns->ms) { 0.00 : ffff8000107d0914: ldrh w2, [x0, #84] 0.00 : ffff8000107d0918: cbz w2, ffff8000107d0964 : if (!blk_integrity_rq(req)) { 0.00 : ffff8000107d091c: ldr w4, [x20, #24] 0.00 : ffff8000107d0920: ldrb w0, [x0, #93] 0.00 : ffff8000107d0924: tbnz w4, #16, ffff8000107d0948 : nvme_ns_has_pi(): : return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple); 0.00 : ffff8000107d0928: cmp w0, #0x0 0.00 : ffff8000107d092c: ccmp w2, #0x8, #0x0, ne // ne = any 0.00 : ffff8000107d0930: b.eq ffff8000107d0940 // b.none : nvme_setup_rw(): : if (WARN_ON_ONCE(!nvme_ns_has_pi(ns))) 0.00 : ffff8000107d0934: brk #0x800 : return BLK_STS_NOTSUPP; 0.00 : ffff8000107d0938: mov w1, #0x1 // #1 0.00 : ffff8000107d093c: b ffff8000107d07b8 : control |= NVME_RW_PRINFO_PRACT; 0.00 : ffff8000107d0940: orr w1, w1, #0x2000 0.00 : ffff8000107d0944: and w1, w1, #0xffff : switch (ns->pi_type) { 0.00 : ffff8000107d0948: cbz w0, ffff8000107d0964 0.00 : ffff8000107d094c: cmp w0, #0x2 0.00 : ffff8000107d0950: b.ls ffff8000107d09c8 // b.plast : control |= NVME_RW_PRINFO_PRCHK_GUARD; 0.00 : ffff8000107d0954: orr w2, w1, #0x1000 0.00 : ffff8000107d0958: cmp w0, #0x3 0.00 : ffff8000107d095c: and w0, w2, #0xffff 0.00 : ffff8000107d0960: csel w1, w0, w1, eq // eq = none : cmnd->rw.control = cpu_to_le16(control); 0.00 : ffff8000107d0964: strh w1, [x19, #50] : return 0; 0.00 : ffff8000107d0968: mov w1, #0x0 // #0 : cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 0.00 : ffff8000107d096c: str w3, [x19, #52] 0.00 : ffff8000107d0970: b ffff8000107d07b8 : if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams) 0.00 : ffff8000107d0974: ldrh w4, [x5, #1232] 0.00 : ffff8000107d0978: cbz w4, ffff8000107d0914 : nvme_assign_write_stream(): : enum rw_hint streamid = req->write_hint; 0.00 : ffff8000107d097c: ldrh w2, [x20, #198] : if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE) 0.00 : ffff8000107d0980: cmp w2, #0x1 0.00 : ffff8000107d0984: b.ls ffff8000107d0a18 // b.plast : streamid--; 0.00 : ffff8000107d0988: sub w2, w2, #0x1 : if (WARN_ON_ONCE(streamid > ctrl->nr_streams)) 0.00 : ffff8000107d098c: cmp w2, w4 0.00 : ffff8000107d0990: b.hi ffff8000107d0a20 // b.pmore : *control |= NVME_RW_DTYPE_STREAMS; 0.00 : ffff8000107d0994: orr w1, w1, #0x10 : *dsmgmt |= streamid << 16; 0.00 : ffff8000107d0998: orr w3, w3, w2, lsl #16 : *control |= NVME_RW_DTYPE_STREAMS; 0.00 : ffff8000107d099c: and w1, w1, #0xffff : if (streamid < ARRAY_SIZE(req->q->write_hints)) 0.00 : ffff8000107d09a0: cmp w2, #0x4 0.00 : ffff8000107d09a4: b.hi ffff8000107d0914 // b.pmore 0.00 : ffff8000107d09a8: ldr x5, [x20] : req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9; 0.00 : ffff8000107d09ac: ldr w4, [x20, #40] 0.00 : ffff8000107d09b0: add x2, x5, w2, uxtw #3 0.00 : ffff8000107d09b4: lsr w5, w4, #9 0.00 : ffff8000107d09b8: ldr x4, [x2, #1992] 0.00 : ffff8000107d09bc: add x4, x4, x5 0.00 : ffff8000107d09c0: str x4, [x2, #1992] 0.00 : ffff8000107d09c4: b ffff8000107d0914 : t10_pi_ref_tag(): : #define T10_PI_APP_ESCAPE cpu_to_be16(0xffff) : #define T10_PI_REF_ESCAPE cpu_to_be32(0xffffffff) : : static inline u32 t10_pi_ref_tag(struct request *rq) : { : unsigned int shift = ilog2(queue_logical_block_size(rq->q)); 0.00 : ffff8000107d09c8: ldr x4, [x20] : nvme_setup_rw(): : control |= NVME_RW_PRINFO_PRCHK_GUARD | 0.00 : ffff8000107d09cc: mov w0, #0x1400 // #5120 0.00 : ffff8000107d09d0: orr w1, w1, w0 : queue_logical_block_size(): : : static inline unsigned queue_logical_block_size(const struct request_queue *q) : { : int retval = 512; : : if (q && q->limits.logical_block_size) 0.00 : ffff8000107d09d4: mov w0, #0x9 // #9 : nvme_setup_rw(): 0.00 : ffff8000107d09d8: and w1, w1, #0xffff : queue_logical_block_size(): 0.00 : ffff8000107d09dc: cbz x4, ffff8000107d09f8 0.00 : ffff8000107d09e0: ldr w5, [x4, #1088] 0.00 : ffff8000107d09e4: clz w2, w5 0.00 : ffff8000107d09e8: cmp w5, #0x0 0.00 : ffff8000107d09ec: mvn w2, w2 0.00 : ffff8000107d09f0: add w2, w2, #0x20 0.00 : ffff8000107d09f4: csel w0, w2, w0, ne // ne = any : t10_pi_ref_tag(): : : #ifdef CONFIG_BLK_DEV_INTEGRITY : if (rq->q->integrity.interval_exp) 0.00 : ffff8000107d09f8: ldrb w4, [x4, #210] : shift = rq->q->integrity.interval_exp; : #endif : return blk_rq_pos(rq) >> (shift - SECTOR_SHIFT) & 0xffffffff; 0.00 : ffff8000107d09fc: ldr x2, [x20, #48] : shift = rq->q->integrity.interval_exp; 0.00 : ffff8000107d0a00: cmp w4, #0x0 0.00 : ffff8000107d0a04: csel w0, w0, w4, eq // eq = none : return blk_rq_pos(rq) >> (shift - SECTOR_SHIFT) & 0xffffffff; 0.00 : ffff8000107d0a08: sub w0, w0, #0x9 0.00 : ffff8000107d0a0c: lsr x0, x2, x0 0.00 : ffff8000107d0a10: str w0, [x19, #56] 0.00 : ffff8000107d0a14: b ffff8000107d0964 : nvme_assign_write_stream(): : streamid = 0; 0.00 : ffff8000107d0a18: mov w2, #0x0 // #0 0.00 : ffff8000107d0a1c: b ffff8000107d09a8 : if (WARN_ON_ONCE(streamid > ctrl->nr_streams)) 0.00 : ffff8000107d0a20: brk #0x800 0.00 : ffff8000107d0a24: b ffff8000107d0914 Percent | Source code & Disassembly of vmlinux for cycles (81 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010457540 : : bio_iov_iter_get_pages(): : * fit into the bio, or are requested in *iter, whatever is smaller. If : * MM encounters an error pinning the requested pages, it stops. Error : * is returned only if 0 pages could be pinned. : */ : int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) : { 1.25 : ffff800010457540: stp x29, x30, [sp, #-144]! 0.00 : ffff800010457544: mov x29, sp 9.77 : ffff800010457548: str x22, [sp, #40] 0.00 : ffff80001045754c: mov x22, x1 0.00 : ffff800010457550: str x28, [sp, #88] 0.00 : ffff800010457554: mov x28, x0 0.00 : ffff800010457558: stp x25, x26, [sp, #64] 0.00 : ffff80001045755c: adrp x0, ffff800011909000 0.00 : ffff800010457560: add x0, x0, #0x908 4.99 : ffff800010457564: ldr x1, [x0] 29.51 : ffff800010457568: str x1, [x29, #136] 0.00 : ffff80001045756c: mov x1, #0x0 // #0 : iov_iter_type(): : }; : }; : : static inline enum iter_type iov_iter_type(const struct iov_iter *i) : { : return i->type & ~(READ | WRITE); 0.00 : ffff800010457570: ldr w25, [x22] : bio_iov_iter_get_pages(): : const bool is_bvec = iov_iter_is_bvec(iter); : int ret; : : if (WARN_ON_ONCE(bio->bi_vcnt)) 0.00 : ffff800010457574: ldrh w0, [x28, #96] : iov_iter_type(): 0.00 : ffff800010457578: and w1, w25, #0xfffffffe 0.00 : ffff80001045757c: str w1, [x29, #108] : bio_iov_iter_get_pages(): 1.25 : ffff800010457580: cbnz w0, ffff8000104577dc 0.00 : ffff800010457584: stp x19, x20, [x29, #16] 2.50 : ffff800010457588: str x21, [x29, #32] 0.00 : ffff80001045758c: stp x23, x24, [x29, #48] : __ll_sc_atomic_sub_return(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010457590: mov w23, #0x1 // #1 0.00 : ffff800010457594: str x27, [x29, #80] : bio_iov_iter_get_pages(): : return -EINVAL; : : do { : if (is_bvec) 7.42 : ffff800010457598: ldr w0, [x29, #108] 0.00 : ffff80001045759c: cmp w0, #0x10 0.00 : ffff8000104575a0: b.eq ffff800010457768 // b.none : __bio_iov_iter_get_pages(): : unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 2.50 : ffff8000104575a4: ldrh w5, [x28, #96] : size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 0.00 : ffff8000104575a8: add x4, x29, #0x80 : unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 0.00 : ffff8000104575ac: ldrh w1, [x28, #98] : size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 0.00 : ffff8000104575b0: mov x2, #0x7fffffffffffffff // #9223372036854775807 : struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; 0.00 : ffff8000104575b4: ldr x20, [x28, #104] : size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 0.00 : ffff8000104575b8: mov x0, x22 : unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 0.00 : ffff8000104575bc: sub w1, w1, w5 : bool same_page = false; 0.00 : ffff8000104575c0: strb wzr, [x29, #127] : unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 0.00 : ffff8000104575c4: and w1, w1, #0xffff : size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 0.00 : ffff8000104575c8: mov w3, w1 : pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); 0.00 : ffff8000104575cc: ubfiz x1, x1, #3, #16 0.00 : ffff8000104575d0: add x1, x1, w5, uxtw #4 0.00 : ffff8000104575d4: add x20, x20, x1 : size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 0.00 : ffff8000104575d8: mov x1, x20 0.00 : ffff8000104575dc: bl ffff800010488a68 0.00 : ffff8000104575e0: mov x24, x0 : if (unlikely(size <= 0)) 0.00 : ffff8000104575e4: cmp x0, #0x0 0.00 : ffff8000104575e8: b.le ffff8000104577b8 0.00 : ffff8000104575ec: mov x27, x0 : for (left = size, i = 0; left > 0; left -= len, i++) { 0.00 : ffff8000104575f0: mov w26, #0x0 // #0 : len = min_t(size_t, PAGE_SIZE - offset, left); 0.00 : ffff8000104575f4: mov x21, #0x1000 // #4096 0.00 : ffff8000104575f8: ldr x3, [x29, #128] 0.00 : ffff8000104575fc: b ffff800010457620 : if (same_page) 0.00 : ffff800010457600: ldrb w0, [x29, #127] 0.00 : ffff800010457604: cbnz w0, ffff800010457720 : offset = 0; 0.00 : ffff800010457608: str xzr, [x29, #128] : for (left = size, i = 0; left > 0; left -= len, i++) { 0.00 : ffff80001045760c: sub x27, x27, w19, uxtw 0.00 : ffff800010457610: add w26, w26, #0x1 0.00 : ffff800010457614: mov x3, #0x0 // #0 0.00 : ffff800010457618: cmp x27, #0x0 0.00 : ffff80001045761c: b.le ffff8000104576e8 : len = min_t(size_t, PAGE_SIZE - offset, left); 0.00 : ffff800010457620: sub x19, x21, x3 : struct page *page = pages[i]; 0.00 : ffff800010457624: ldr x25, [x20, w26, uxtw #3] : len = min_t(size_t, PAGE_SIZE - offset, left); 0.00 : ffff800010457628: cmp x19, x27 : if (__bio_try_merge_page(bio, page, len, offset, &same_page)) { 0.00 : ffff80001045762c: add x4, x29, #0x7f : len = min_t(size_t, PAGE_SIZE - offset, left); 0.00 : ffff800010457630: csel x19, x19, x27, ls // ls = plast : if (__bio_try_merge_page(bio, page, len, offset, &same_page)) { 0.00 : ffff800010457634: mov x1, x25 0.00 : ffff800010457638: mov w2, w19 0.00 : ffff80001045763c: mov x0, x28 0.00 : ffff800010457640: bl ffff800010455378 <__bio_try_merge_page> 14.86 : ffff800010457644: tst w0, #0xff 0.00 : ffff800010457648: b.ne ffff800010457600 // b.any : bio_full(): : * Return true if @bio is full and one segment with @len bytes can't be : * added to the bio, otherwise return false : */ : static inline bool bio_full(struct bio *bio, unsigned len) : { : if (bio->bi_vcnt >= bio->bi_max_vecs) 0.00 : ffff80001045764c: ldrh w1, [x28, #96] 0.00 : ffff800010457650: ldrh w0, [x28, #98] 0.00 : ffff800010457654: cmp w1, w0 0.00 : ffff800010457658: b.cc ffff8000104576ac // b.lo, b.ul, b.last : __bio_iov_iter_get_pages(): : if (WARN_ON_ONCE(bio_full(bio, len))) 0.00 : ffff80001045765c: brk #0x800 : return -EINVAL; 0.00 : ffff800010457660: mov w0, #0xffffffea // #-22 : bio_iov_iter_get_pages(): : ret = __bio_iov_iter_get_pages(bio, iter); : } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); : : if (is_bvec) : bio_set_flag(bio, BIO_NO_PAGE_REF); : return bio->bi_vcnt ? 0 : ret; 6.22 : ffff800010457664: ldrh w1, [x28, #96] 1.24 : ffff800010457668: ldp x19, x20, [x29, #16] 0.00 : ffff80001045766c: cmp w1, #0x0 1.18 : ffff800010457670: ldr x21, [x29, #32] 0.00 : ffff800010457674: csel w0, w0, wzr, eq // eq = none 0.00 : ffff800010457678: ldp x23, x24, [x29, #48] 0.00 : ffff80001045767c: ldr x27, [x29, #80] : } 0.00 : ffff800010457680: adrp x1, ffff800011909000 0.00 : ffff800010457684: add x26, x1, #0x908 1.26 : ffff800010457688: ldr x2, [x29, #136] 0.00 : ffff80001045768c: ldr x1, [x26] 0.00 : ffff800010457690: eor x1, x2, x1 0.00 : ffff800010457694: cbnz x1, ffff8000104577f0 0.00 : ffff800010457698: ldr x22, [sp, #40] 3.74 : ffff80001045769c: ldp x25, x26, [sp, #64] 0.00 : ffff8000104576a0: ldr x28, [sp, #88] 0.00 : ffff8000104576a4: ldp x29, x30, [sp], #144 0.00 : ffff8000104576a8: ret : bio_full(): : return true; : : if (bio->bi_iter.bi_size > UINT_MAX - len) 0.00 : ffff8000104576ac: ldr w1, [x28, #40] 0.00 : ffff8000104576b0: mvn w0, w19 0.00 : ffff8000104576b4: cmp w1, w0 0.00 : ffff8000104576b8: b.hi ffff80001045765c // b.pmore : __bio_iov_iter_get_pages(): : __bio_add_page(bio, page, len, offset); 4.98 : ffff8000104576bc: ldr w3, [x29, #128] 0.00 : ffff8000104576c0: mov w2, w19 0.00 : ffff8000104576c4: mov x1, x25 0.00 : ffff8000104576c8: mov x0, x28 : for (left = size, i = 0; left > 0; left -= len, i++) { 4.85 : ffff8000104576cc: sub x27, x27, w19, uxtw 0.00 : ffff8000104576d0: add w26, w26, #0x1 : __bio_add_page(bio, page, len, offset); 0.00 : ffff8000104576d4: bl ffff800010454eb8 <__bio_add_page> : offset = 0; 2.48 : ffff8000104576d8: str xzr, [x29, #128] 0.00 : ffff8000104576dc: mov x3, #0x0 // #0 : for (left = size, i = 0; left > 0; left -= len, i++) { 0.00 : ffff8000104576e0: cmp x27, #0x0 0.00 : ffff8000104576e4: b.gt ffff800010457620 : iov_iter_advance(iter, size); 0.00 : ffff8000104576e8: mov x1, x24 0.00 : ffff8000104576ec: mov x0, x22 0.00 : ffff8000104576f0: bl ffff800010489c28 : bio_iov_iter_get_pages(): : } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); 0.00 : ffff8000104576f4: ldr x0, [x22, #16] 0.00 : ffff8000104576f8: ldrh w1, [x28, #96] 0.00 : ffff8000104576fc: cbz x0, ffff80001045770c : bio_full(): : if (bio->bi_vcnt >= bio->bi_max_vecs) 0.00 : ffff800010457700: ldrh w0, [x28, #98] 0.00 : ffff800010457704: cmp w0, w1 0.00 : ffff800010457708: b.hi ffff800010457598 // b.pmore : bio_iov_iter_get_pages(): : if (is_bvec) 0.00 : ffff80001045770c: ldr w1, [x29, #108] 0.00 : ffff800010457710: mov w0, #0x0 // #0 0.00 : ffff800010457714: cmp w1, #0x10 0.00 : ffff800010457718: b.ne ffff800010457664 // b.any 0.00 : ffff80001045771c: b ffff8000104577cc : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010457720: ldr x0, [x25, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff800010457724: sub x1, x0, #0x1 0.00 : ffff800010457728: tst x0, #0x1 0.00 : ffff80001045772c: csel x25, x1, x25, ne // ne = any : page_ref_dec_and_test(): : return ret; : } : : static inline int page_ref_dec_and_test(struct page *page) : { : int ret = atomic_dec_and_test(&page->_refcount); 0.00 : ffff800010457730: add x0, x25, #0x34 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010457734: b ffff80001045775c 0.00 : ffff800010457738: b ffff80001045775c : __lse_atomic_sub_return(): : } : : ATOMIC_OP_SUB_RETURN(_relaxed, ) : ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC_OP_SUB_RETURN(_release, l, "memory") : ATOMIC_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001045773c: mov w1, w23 0.00 : ffff800010457740: neg w1, w1 0.00 : ffff800010457744: ldaddal w1, w2, [x0] 0.00 : ffff800010457748: add w1, w1, w2 : put_page(): : if (page_is_devmap_managed(page)) { : put_devmap_managed_page(page); : return; : } : : if (put_page_testzero(page)) 0.00 : ffff80001045774c: cbnz w1, ffff800010457608 : __put_page(page); 0.00 : ffff800010457750: mov x0, x25 0.00 : ffff800010457754: bl ffff8000101dfec0 <__put_page> 0.00 : ffff800010457758: b ffff800010457608 : __ll_sc_atomic_sub_return(): 0.00 : ffff80001045775c: b ffff800010458bb4 : put_page(): : if (put_page_testzero(page)) 0.00 : ffff800010457760: cbnz w1, ffff800010457608 0.00 : ffff800010457764: b ffff800010457750 : __bio_iov_bvec_add_pages(): : const struct bio_vec *bv = iter->bvec; 0.00 : ffff800010457768: ldr x3, [x22, #24] : if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len)) 0.00 : ffff80001045776c: ldr x2, [x22, #8] 0.00 : ffff800010457770: ldr w0, [x3, #8] 0.00 : ffff800010457774: cmp x2, x0 0.00 : ffff800010457778: b.hi ffff8000104577c4 // b.pmore : len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count); 0.00 : ffff80001045777c: ldr x4, [x22, #16] 0.00 : ffff800010457780: sub x19, x0, x2 : size = bio_add_page(bio, bv->bv_page, len, 0.00 : ffff800010457784: ldr x1, [x3] 0.00 : ffff800010457788: mov x0, x28 : len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count); 0.00 : ffff80001045778c: cmp x19, x4 : size = bio_add_page(bio, bv->bv_page, len, 0.00 : ffff800010457790: ldr w3, [x3, #12] : len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count); 0.00 : ffff800010457794: csel x19, x19, x4, ls // ls = plast : size = bio_add_page(bio, bv->bv_page, len, 0.00 : ffff800010457798: add w3, w3, w2 0.00 : ffff80001045779c: mov w2, w19 0.00 : ffff8000104577a0: bl ffff8000104554d0 0.00 : ffff8000104577a4: sxtw x1, w0 : if (unlikely(size != len)) 0.00 : ffff8000104577a8: cmp x1, w19, uxtw 0.00 : ffff8000104577ac: b.eq ffff8000104576ec // b.none 0.00 : ffff8000104577b0: mov w0, #0xffffffea // #-22 0.00 : ffff8000104577b4: b ffff8000104577cc : __bio_iov_iter_get_pages(): : return size ? size : -EFAULT; 0.00 : ffff8000104577b8: b.eq ffff8000104577e8 // b.none : bio_iov_iter_get_pages(): : } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); 0.00 : ffff8000104577bc: cbz w0, ffff8000104576f4 0.00 : ffff8000104577c0: b ffff800010457664 : __bio_iov_bvec_add_pages(): : if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len)) 0.00 : ffff8000104577c4: brk #0x800 0.00 : ffff8000104577c8: mov w0, #0xffffffea // #-22 : bio_set_flag(): : return (bio->bi_flags & (1U << bit)) != 0; : } : : static inline void bio_set_flag(struct bio *bio, unsigned int bit) : { : bio->bi_flags |= (1U << bit); 0.00 : ffff8000104577cc: ldrh w1, [x28, #20] 0.00 : ffff8000104577d0: orr w1, w1, #0x1 0.00 : ffff8000104577d4: strh w1, [x28, #20] 0.00 : ffff8000104577d8: b ffff800010457664 : bio_iov_iter_get_pages(): : if (WARN_ON_ONCE(bio->bi_vcnt)) 0.00 : ffff8000104577dc: brk #0x800 : return -EINVAL; 0.00 : ffff8000104577e0: mov w0, #0xffffffea // #-22 0.00 : ffff8000104577e4: b ffff800010457680 : __bio_iov_iter_get_pages(): : return size ? size : -EFAULT; 0.00 : ffff8000104577e8: mov w0, #0xfffffff2 // #-14 0.00 : ffff8000104577ec: b ffff800010457664 0.00 : ffff8000104577f0: stp x19, x20, [x29, #16] 0.00 : ffff8000104577f4: str x21, [x29, #32] 0.00 : ffff8000104577f8: stp x23, x24, [x29, #48] 0.00 : ffff8000104577fc: str x27, [x29, #80] : bio_iov_iter_get_pages(): : } 0.00 : ffff800010457800: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (29 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010160760 : : rcu_core(): : } : EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); : : /* Perform RCU core processing work for the current CPU. */ : static __latent_entropy void rcu_core(void) : { 6.87 : ffff800010160760: stp x29, x30, [sp, #-128]! 0.00 : ffff800010160764: mov x29, sp 3.43 : ffff800010160768: stp x19, x20, [sp, #16] : unsigned long flags; : struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); 0.00 : ffff80001016076c: adrp x20, ffff800011529000 : { 0.00 : ffff800010160770: stp x21, x22, [sp, #32] : struct rcu_node *rnp = rdp->mynode; : const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) && : rcu_segcblist_is_offloaded(&rdp->cblist); : : if (cpu_is_offline(smp_processor_id())) 0.00 : ffff800010160774: adrp x22, ffff80001151d000 : { 0.00 : ffff800010160778: stp x23, x24, [sp, #48] : if (cpu_is_offline(smp_processor_id())) 0.00 : ffff80001016077c: add x0, x22, #0x18 : { 0.00 : ffff800010160780: str x26, [sp, #72] 0.00 : ffff800010160784: adrp x21, ffff800011909000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 3.28 : ffff800010160788: mrs x24, tpidr_el1 : rcu_core(): : if (cpu_is_offline(smp_processor_id())) 0.00 : ffff80001016078c: ldr w1, [x0, x24] : { 0.00 : ffff800010160790: add x0, x21, #0x908 0.00 : ffff800010160794: ldr x2, [x0] 0.00 : ffff800010160798: str x2, [x29, #120] 0.00 : ffff80001016079c: mov x2, #0x0 // #0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000101607a0: adrp x23, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101607a4: add w0, w1, #0x3f 0.00 : ffff8000101607a8: cmp w1, #0x0 0.00 : ffff8000101607ac: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff8000101607b0: add x2, x23, #0x160 : rcu_core(): : struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); 0.00 : ffff8000101607b4: add x20, x20, #0xac0 : test_bit(): 0.00 : ffff8000101607b8: asr w0, w0, #6 0.00 : ffff8000101607bc: sxtw x0, w0 : rcu_core(): 0.00 : ffff8000101607c0: add x19, x20, x24 : test_bit(): 0.00 : ffff8000101607c4: ldr x0, [x2, x0, lsl #3] : rcu_core(): : struct rcu_node *rnp = rdp->mynode; 0.00 : ffff8000101607c8: ldr x26, [x19, #24] : test_bit(): 0.00 : ffff8000101607cc: lsr x1, x0, x1 : rcu_core(): : if (cpu_is_offline(smp_processor_id())) 0.00 : ffff8000101607d0: tbz w1, #0, ffff8000101608a0 0.00 : ffff8000101607d4: str x25, [x29, #64] : return; : trace_rcu_utilization(TPS("Start RCU core")); : WARN_ON_ONCE(!rdp->beenonline); 3.52 : ffff8000101607d8: ldrb w0, [x19, #19] 0.00 : ffff8000101607dc: cbz w0, ffff800010160b14 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000101607e0: mrs x3, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101607e4: ldr w0, [x3, #16] : rcu_core(): : : /* Report any deferred quiescent states if preemption enabled. */ : if (!(preempt_count() & PREEMPT_MASK)) { 0.00 : ffff8000101607e8: and w0, w0, #0xff 0.00 : ffff8000101607ec: cbz w0, ffff8000101608cc : rcu_preempt_deferred_qs(current); : } else if (rcu_preempt_need_deferred_qs(current)) { 0.00 : ffff8000101607f0: mov x0, x3 0.00 : ffff8000101607f4: bl ffff80001015df30 0.00 : ffff8000101607f8: tst w0, #0xff 0.00 : ffff8000101607fc: b.ne ffff800010160ac4 // b.any : rcu_check_quiescent_state(): : note_gp_changes(rdp); 3.49 : ffff800010160800: mov x0, x19 0.00 : ffff800010160804: bl ffff80001015f008 : if (!rdp->core_needs_qs) 0.00 : ffff800010160808: ldrb w0, [x19, #18] 0.00 : ffff80001016080c: cbz w0, ffff800010160818 : if (rdp->cpu_no_qs.b.norm) 0.00 : ffff800010160810: ldrb w0, [x19, #16] 0.00 : ffff800010160814: cbz w0, ffff8000101608f4 : __read_once_size(): 0.00 : ffff800010160818: adrp x25, ffff800011922000 0.00 : ffff80001016081c: add x0, x25, #0xc80 0.00 : ffff800010160820: ldr x0, [x0, #8776] : rcu_core(): : : /* Update RCU state based on any recent quiescent states. */ : rcu_check_quiescent_state(rdp); : : /* No grace period and unregistered callbacks? */ : if (!rcu_gp_in_progress() && 0.00 : ffff800010160824: tst x0, #0x3 0.00 : ffff800010160828: b.ne ffff800010160834 // b.any 0.00 : ffff80001016082c: ldrb w0, [x19, #160] 0.00 : ffff800010160830: cbnz w0, ffff800010160ae0 : __read_once_size(): 24.11 : ffff800010160834: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010160838: ldr w1, [x0, #660] : rcu_jiffies_till_stall_check(): : : /* : * Limit check must be consistent with the Kconfig limits : * for CONFIG_RCU_CPU_STALL_TIMEOUT. : */ : if (till_stall_check < 3) { 0.00 : ffff80001016083c: cmp w1, #0x2 0.00 : ffff800010160840: b.gt ffff8000101608e0 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010160844: mov w1, #0x3 // #3 0.00 : ffff800010160848: str w1, [x0, #660] : rcu_core(): : } : : rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); : : /* If there are callbacks ready, invoke them. */ : if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) && 0.00 : ffff80001016084c: add x24, x19, #0x50 0.00 : ffff800010160850: mov x0, x24 0.00 : ffff800010160854: bl ffff800010162b90 0.00 : ffff800010160858: tst w0, #0xff 0.00 : ffff80001016085c: b.eq ffff8000101608d8 // b.none : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010160860: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010160864: add x0, x0, #0x2a0 0.00 : ffff800010160868: ldr w0, [x0, #8] : rcu_core(): 0.00 : ffff80001016086c: cbz w0, ffff8000101608d8 : rcu_do_batch(): : struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl); 0.00 : ffff800010160870: stp xzr, xzr, [x29, #96] 0.00 : ffff800010160874: add x20, x29, #0x60 : if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { 0.00 : ffff800010160878: mov x0, x24 : struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl); 0.00 : ffff80001016087c: stp x20, xzr, [x29, #104] : if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { 0.00 : ffff800010160880: bl ffff800010162b90 0.00 : ffff800010160884: tst w0, #0xff 0.00 : ffff800010160888: b.ne ffff80001016093c // b.any : __read_once_size(): 0.00 : ffff80001016088c: ldr x0, [x19, #152] 0.00 : ffff800010160890: ldr x0, [x19, #80] 0.00 : ffff800010160894: ldr x25, [x29, #64] : get_current(): 0.00 : ffff800010160898: mrs x0, sp_el0 : test_bit(): 0.00 : ffff80001016089c: ldr x0, [x0] : rcu_core(): : rcu_do_batch(rdp); : : /* Do any needed deferred wakeups of rcuo kthreads. */ : do_nocb_deferred_wakeup(rdp); : trace_rcu_utilization(TPS("End RCU core")); : } 16.58 : ffff8000101608a0: add x21, x21, #0x908 0.00 : ffff8000101608a4: ldr x1, [x29, #120] 0.00 : ffff8000101608a8: ldr x0, [x21] 0.00 : ffff8000101608ac: eor x0, x1, x0 0.00 : ffff8000101608b0: cbnz x0, ffff800010160c18 0.00 : ffff8000101608b4: ldp x19, x20, [sp, #16] 3.54 : ffff8000101608b8: ldp x21, x22, [sp, #32] 0.00 : ffff8000101608bc: ldp x23, x24, [sp, #48] 0.00 : ffff8000101608c0: ldr x26, [sp, #72] 0.00 : ffff8000101608c4: ldp x29, x30, [sp], #128 0.00 : ffff8000101608c8: ret : rcu_preempt_deferred_qs(current); 0.00 : ffff8000101608cc: mov x0, x3 0.00 : ffff8000101608d0: bl ffff800010160210 0.00 : ffff8000101608d4: b ffff800010160800 0.00 : ffff8000101608d8: ldr x25, [x29, #64] 0.00 : ffff8000101608dc: b ffff8000101608a0 : rcu_jiffies_till_stall_check(): : WRITE_ONCE(rcu_cpu_stall_timeout, 3); : till_stall_check = 3; : } else if (till_stall_check > 300) { 0.00 : ffff8000101608e0: cmp w1, #0x12c 0.00 : ffff8000101608e4: b.le ffff80001016084c : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101608e8: mov w1, #0x12c // #300 0.00 : ffff8000101608ec: str w1, [x0, #660] 0.00 : ffff8000101608f0: b ffff80001016084c 0.00 : ffff8000101608f4: str x27, [x29, #80] : rcu_report_qs_rdp(): : rnp = rdp->mynode; 0.00 : ffff8000101608f8: ldr x25, [x19, #24] : raw_spin_lock_irqsave_rcu_node(rnp, flags); 0.00 : ffff8000101608fc: mov x0, x25 0.00 : ffff800010160900: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff800010160904: mov x27, x0 : if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq || 0.00 : ffff800010160908: ldrb w0, [x19, #16] 0.00 : ffff80001016090c: cbnz w0, ffff800010160920 0.00 : ffff800010160910: ldr x1, [x20, x24] 10.75 : ffff800010160914: ldr x0, [x25, #8] 0.00 : ffff800010160918: cmp x1, x0 0.00 : ffff80001016091c: b.eq ffff800010160b90 // b.none : rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */ 3.34 : ffff800010160920: mov w0, #0x1 // #1 0.00 : ffff800010160924: strb w0, [x19, #16] : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff800010160928: mov x1, x27 0.00 : ffff80001016092c: mov x0, x25 0.00 : ffff800010160930: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010160934: ldr x27, [x29, #80] 0.00 : ffff800010160938: b ffff800010160818 0.00 : ffff80001016093c: stp x27, x28, [x29, #80] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010160940: mrs x27, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010160944: and w0, w27, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff800010160948: cbnz w0, ffff800010160954 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001016094c: mov x0, #0x60 // #96 0.00 : ffff800010160950: msr daifset, #0x2 : __my_cpu_offset(): 0.00 : ffff800010160954: mrs x0, tpidr_el1 : rcu_do_batch(): : WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); 0.00 : ffff800010160958: add x22, x22, #0x18 : test_bit(): 0.00 : ffff80001016095c: add x23, x23, #0x160 : rcu_do_batch(): 0.00 : ffff800010160960: ldr w1, [x22, x0] : test_bit(): 0.00 : ffff800010160964: add w0, w1, #0x3f 0.00 : ffff800010160968: cmp w1, #0x0 0.00 : ffff80001016096c: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff800010160970: asr w0, w0, #6 0.00 : ffff800010160974: sxtw x0, w0 0.00 : ffff800010160978: ldr x0, [x23, x0, lsl #3] 0.00 : ffff80001016097c: lsr x1, x0, x1 : rcu_do_batch(): 0.00 : ffff800010160980: tbz w1, #0, ffff800010160bf0 : bl = max(rdp->blimit, pending >> rcu_divisor); 0.00 : ffff800010160984: adrp x26, ffff800011922000 0.00 : ffff800010160988: add x28, x26, #0xb00 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001016098c: ldr x22, [x19, #152] : rcu_do_batch(): : long pending, tlimit = 0; 0.00 : ffff800010160990: mov x23, #0x0 // #0 : bl = max(rdp->blimit, pending >> rcu_divisor); 0.00 : ffff800010160994: ldr x0, [x19, #184] 0.00 : ffff800010160998: ldr w1, [x28, #192] 0.00 : ffff80001016099c: asr x22, x22, x1 0.00 : ffff8000101609a0: cmp x22, x0 0.00 : ffff8000101609a4: csel x22, x22, x0, ge // ge = tcont : if (unlikely(bl > 100)) 0.00 : ffff8000101609a8: cmp x22, #0x64 0.00 : ffff8000101609ac: b.gt ffff800010160be0 : __read_once_size(): 0.00 : ffff8000101609b0: ldr x0, [x19, #152] : rcu_do_batch(): : rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); 0.00 : ffff8000101609b4: mov x1, x20 0.00 : ffff8000101609b8: mov x0, x24 0.00 : ffff8000101609bc: bl ffff800010162d20 : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff8000101609c0: msr daif, x27 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101609c4: nop : rcu_do_batch(): : rhp = rcu_cblist_dequeue(&rcl); 0.00 : ffff8000101609c8: mov x0, x20 0.00 : ffff8000101609cc: bl ffff800010162ae8 : for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) { 0.00 : ffff8000101609d0: cbz x0, ffff800010160a20 : f = rhp->func; 0.00 : ffff8000101609d4: ldr x1, [x0, #8] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101609d8: str xzr, [x0, #8] : rcu_do_batch(): : f(rhp); 0.00 : ffff8000101609dc: blr x1 : if (-rcl.len >= bl && !offloaded && 0.00 : ffff8000101609e0: ldr x0, [x29, #112] 0.00 : ffff8000101609e4: neg x0, x0 0.00 : ffff8000101609e8: cmp x22, x0 0.00 : ffff8000101609ec: b.gt ffff800010160a04 : get_current(): 0.00 : ffff8000101609f0: mrs x1, sp_el0 : test_bit(): 0.00 : ffff8000101609f4: ldr x2, [x1] : rcu_do_batch(): 0.00 : ffff8000101609f8: tbnz w2, #1, ffff800010160a20 : is_idle_task(): : * : * Return: 1 if @p is an idle task. 0 otherwise. : */ : static inline bool is_idle_task(const struct task_struct *p) : { : return !!(p->flags & PF_IDLE); 0.00 : ffff8000101609fc: ldr w1, [x1, #44] : rcu_do_batch(): : (need_resched() || 0.00 : ffff800010160a00: tbz w1, #1, ffff800010160a20 : if (unlikely(tlimit)) { 0.00 : ffff800010160a04: cbz x23, ffff8000101609c8 : if (likely((-rcl.len & 31) || local_clock() < tlimit)) 0.00 : ffff800010160a08: tst x0, #0x1f 0.00 : ffff800010160a0c: b.ne ffff8000101609c8 // b.any : local_clock(): : return sched_clock(); : } : : static inline u64 local_clock(void) : { : return sched_clock(); 0.00 : ffff800010160a10: bl ffff80001017af20 : rcu_do_batch(): 0.00 : ffff800010160a14: cmp x0, x23 0.00 : ffff800010160a18: b.cc ffff8000101609c8 // b.lo, b.ul, b.last 0.00 : ffff800010160a1c: nop : arch_local_save_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160a20: mrs x22, daif : arch_irqs_disabled_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160a24: and w0, w22, #0x80 : arch_local_irq_save(): : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff800010160a28: cbnz w0, ffff800010160a34 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160a2c: mov x0, #0x60 // #96 0.00 : ffff800010160a30: msr daifset, #0x2 : get_current(): 0.00 : ffff800010160a34: mrs x0, sp_el0 : test_bit(): 0.00 : ffff800010160a38: ldr x0, [x0] : rcu_do_batch(): : rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); 0.00 : ffff800010160a3c: mov x1, x20 0.00 : ffff800010160a40: mov x0, x24 0.00 : ffff800010160a44: bl ffff800010162df0 : smp_mb(); /* List handling before counting for rcu_barrier(). */ 0.00 : ffff800010160a48: dmb ish : rcu_segcblist_insert_count(&rdp->cblist, &rcl); 0.00 : ffff800010160a4c: mov x1, x20 0.00 : ffff800010160a50: mov x0, x24 0.00 : ffff800010160a54: bl ffff800010162dd0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010160a58: ldr x0, [x19, #152] : rcu_do_batch(): : if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark) 0.00 : ffff800010160a5c: mov x1, #0x270f // #9999 0.00 : ffff800010160a60: ldr x2, [x19, #184] 0.00 : ffff800010160a64: cmp x2, x1 0.00 : ffff800010160a68: b.le ffff800010160a7c 0.00 : ffff800010160a6c: add x1, x26, #0xb00 0.00 : ffff800010160a70: ldr x2, [x1, #208] 0.00 : ffff800010160a74: cmp x0, x2 0.00 : ffff800010160a78: b.le ffff800010160b74 0.00 : ffff800010160a7c: ldr x1, [x19, #168] : if (count == 0 && rdp->qlen_last_fqs_check != 0) { 0.00 : ffff800010160a80: cbnz x0, ffff800010160b1c 0.00 : ffff800010160a84: cbz x1, ffff800010160a98 : rdp->n_force_qs_snap = rcu_state.n_force_qs; 0.00 : ffff800010160a88: add x25, x25, #0xc80 : rdp->qlen_last_fqs_check = 0; 0.00 : ffff800010160a8c: str xzr, [x19, #168] : rdp->n_force_qs_snap = rcu_state.n_force_qs; 0.00 : ffff800010160a90: ldr x0, [x25, #9048] 0.00 : ffff800010160a94: str x0, [x19, #176] : __read_once_size(): 0.00 : ffff800010160a98: ldr x0, [x19, #80] : rcu_do_batch(): : WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist)); 0.00 : ffff800010160a9c: cbnz x0, ffff800010160b6c : arch_local_irq_restore(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160aa0: msr daif, x22 : arch_static_branch(): 0.00 : ffff800010160aa4: nop : rcu_do_batch(): : if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist)) 0.00 : ffff800010160aa8: mov x0, x24 0.00 : ffff800010160aac: bl ffff800010162b90 0.00 : ffff800010160ab0: tst w0, #0xff 0.00 : ffff800010160ab4: b.ne ffff800010160b5c // b.any 0.00 : ffff800010160ab8: ldr x25, [x29, #64] 0.00 : ffff800010160abc: ldp x27, x28, [x29, #80] 0.00 : ffff800010160ac0: b ffff8000101608a0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010160ac4: b ffff800010160b0c 0.00 : ffff800010160ac8: b ffff800010160b0c : __lse_atomic64_or(): : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) 0.00 : ffff800010160acc: mov x0, #0x2 // #2 0.00 : ffff800010160ad0: stset x0, [x3] : get_current(): 0.00 : ffff800010160ad4: mrs x0, sp_el0 : set_preempt_need_resched(): : task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \ : } while (0) : : static inline void set_preempt_need_resched(void) : { : current_thread_info()->preempt.need_resched = 0; 0.00 : ffff800010160ad8: str wzr, [x0, #20] 0.00 : ffff800010160adc: b ffff800010160800 : arch_local_save_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160ae0: mrs x20, daif : arch_irqs_disabled_flags(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160ae4: and w0, w20, #0x80 : arch_local_irq_save(): : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff800010160ae8: cbnz w0, ffff800010160af4 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160aec: mov x0, #0x60 // #96 3.51 : ffff800010160af0: msr daifset, #0x2 : __read_once_size(): 0.00 : ffff800010160af4: ldr x0, [x19, #104] 0.00 : ffff800010160af8: ldr x0, [x0] : rcu_core(): : if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) 0.00 : ffff800010160afc: cbnz x0, ffff800010160b80 : arch_local_irq_restore(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010160b00: msr daif, x20 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010160b04: nop 0.00 : ffff800010160b08: b ffff800010160834 : __ll_sc_atomic64_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010160b0c: b ffff80001016260c 0.00 : ffff800010160b10: b ffff800010160ad4 : rcu_core(): : WARN_ON_ONCE(!rdp->beenonline); 0.00 : ffff800010160b14: brk #0x800 0.00 : ffff800010160b18: b ffff8000101607e0 : rcu_do_batch(): : } else if (count < rdp->qlen_last_fqs_check - qhimark) 0.00 : ffff800010160b1c: add x26, x26, #0xb00 0.00 : ffff800010160b20: ldr x2, [x26, #184] 0.00 : ffff800010160b24: sub x1, x1, x2 0.00 : ffff800010160b28: cmp x1, x0 0.00 : ffff800010160b2c: b.le ffff800010160b34 : rdp->qlen_last_fqs_check = count; 0.00 : ffff800010160b30: str x0, [x19, #168] : __read_once_size(): 0.00 : ffff800010160b34: ldr x0, [x19, #80] : rcu_do_batch(): : WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) && 0.00 : ffff800010160b38: cbnz x0, ffff800010160aa0 0.00 : ffff800010160b3c: brk #0x800 0.00 : ffff800010160b40: b ffff800010160aa0 : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff800010160b44: dsb sy 0.00 : ffff800010160b48: b ffff8000101609c8 0.00 : ffff800010160b4c: dsb sy 0.00 : ffff800010160b50: b ffff800010160aa8 0.00 : ffff800010160b54: dsb sy 3.52 : ffff800010160b58: b ffff800010160834 : rcu_do_batch(): : invoke_rcu_core(); 0.00 : ffff800010160b5c: bl ffff80001015eda8 0.00 : ffff800010160b60: ldr x25, [x29, #64] 0.00 : ffff800010160b64: ldp x27, x28, [x29, #80] 0.00 : ffff800010160b68: b ffff8000101608a0 : WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist)); 0.00 : ffff800010160b6c: brk #0x800 0.00 : ffff800010160b70: b ffff800010160aa0 : rdp->blimit = blimit; 0.00 : ffff800010160b74: ldr x1, [x1, #216] 0.00 : ffff800010160b78: str x1, [x19, #184] 0.00 : ffff800010160b7c: b ffff800010160a7c : rcu_core(): : rcu_accelerate_cbs_unlocked(rnp, rdp); 0.00 : ffff800010160b80: mov x1, x19 0.00 : ffff800010160b84: mov x0, x26 0.00 : ffff800010160b88: bl ffff80001015d090 0.00 : ffff800010160b8c: b ffff800010160b00 : rcu_report_qs_rdp(): : if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq || 0.00 : ffff800010160b90: ldrb w0, [x19, #20] 0.00 : ffff800010160b94: cbnz w0, ffff800010160920 : mask = rdp->grpmask; 0.00 : ffff800010160b98: ldr x20, [x19, #32] : if ((rnp->qsmask & mask) == 0) { 0.00 : ffff800010160b9c: ldr x0, [x25, #32] 0.00 : ffff800010160ba0: tst x20, x0 0.00 : ffff800010160ba4: b.eq ffff800010160bf8 // b.none : needwake = rcu_accelerate_cbs(rnp, rdp); 0.00 : ffff800010160ba8: mov x1, x19 0.00 : ffff800010160bac: mov x0, x25 0.00 : ffff800010160bb0: bl ffff80001015c9d8 : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff800010160bb4: strb wzr, [x19, #221] 0.00 : ffff800010160bb8: strb wzr, [x19, #220] : rcu_report_qs_rdp(): 0.00 : ffff800010160bbc: and w24, w0, #0xff : rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); 0.00 : ffff800010160bc0: mov x3, x27 0.00 : ffff800010160bc4: mov x1, x25 0.00 : ffff800010160bc8: ldr x2, [x25, #8] 0.00 : ffff800010160bcc: mov x0, x20 0.00 : ffff800010160bd0: bl ffff80001015cda8 : if (needwake) 0.00 : ffff800010160bd4: cbnz w24, ffff800010160c0c 0.00 : ffff800010160bd8: ldr x27, [x29, #80] 0.00 : ffff800010160bdc: b ffff800010160818 : local_clock(): 0.00 : ffff800010160be0: bl ffff80001017af20 : rcu_do_batch(): : tlimit = local_clock() + rcu_resched_ns; 0.00 : ffff800010160be4: ldr x23, [x28, #200] 0.00 : ffff800010160be8: add x23, x0, x23 0.00 : ffff800010160bec: b ffff8000101609b0 : WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); 0.00 : ffff800010160bf0: brk #0x800 0.00 : ffff800010160bf4: b ffff800010160984 : rcu_report_qs_rdp(): : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 14.06 : ffff800010160bf8: mov x1, x27 0.00 : ffff800010160bfc: mov x0, x25 0.00 : ffff800010160c00: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010160c04: ldr x27, [x29, #80] 0.00 : ffff800010160c08: b ffff800010160818 : rcu_gp_kthread_wake(); 0.00 : ffff800010160c0c: bl ffff80001015ccd0 0.00 : ffff800010160c10: ldr x27, [x29, #80] 0.00 : ffff800010160c14: b ffff800010160818 0.00 : ffff800010160c18: str x25, [x29, #64] 0.00 : ffff800010160c1c: stp x27, x28, [x29, #80] : rcu_core(): : } 0.00 : ffff800010160c20: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (87 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001045b7e0 : : submit_bio(): : * uses that function to do most of the work. Both are fairly rough : * interfaces; @bio must be presetup and ready for I/O. : * : */ : blk_qc_t submit_bio(struct bio *bio) : { 1.16 : ffff80001045b7e0: stp x29, x30, [sp, #-128]! 0.00 : ffff80001045b7e4: mov x29, sp 4.69 : ffff80001045b7e8: stp x19, x20, [sp, #16] 0.00 : ffff80001045b7ec: adrp x19, ffff800011909000 0.00 : ffff80001045b7f0: add x1, x19, #0x908 0.00 : ffff80001045b7f4: mov x20, x0 0.00 : ffff80001045b7f8: ldr x2, [x1] 1.07 : ffff80001045b7fc: str x2, [x29, #120] 0.00 : ffff80001045b800: mov x2, #0x0 // #0 : blkcg_punt_bio_submit(): : : bool __blkcg_punt_bio_submit(struct bio *bio); : : static inline bool blkcg_punt_bio_submit(struct bio *bio) : { : if (bio->bi_opf & REQ_CGROUP_PUNT) 0.00 : ffff80001045b804: ldr w1, [x0, #16] 0.00 : ffff80001045b808: tbnz w1, #23, ffff80001045b848 : bio_has_data(): : /* : * Check whether this bio carries any data or not. A NULL bio is allowed. : */ : static inline bool bio_has_data(struct bio *bio) : { : if (bio && 1.18 : ffff80001045b80c: cbz x20, ffff80001045b818 : bio->bi_iter.bi_size && 2.31 : ffff80001045b810: ldr w0, [x20, #40] : if (bio && 0.00 : ffff80001045b814: cbnz w0, ffff80001045b85c : submit_bio(): : * submission can be a significant part of overall IO time. : */ : if (workingset_read) : psi_memstall_enter(&pflags); : : ret = generic_make_request(bio); 2.04 : ffff80001045b818: mov x0, x20 0.00 : ffff80001045b81c: bl ffff80001045b4c0 1.16 : ffff80001045b820: mov w1, w0 : : if (workingset_read) : psi_memstall_leave(&pflags); : : return ret; : } 0.00 : ffff80001045b824: add x19, x19, #0x908 0.00 : ffff80001045b828: mov w0, w1 1.17 : ffff80001045b82c: ldr x2, [x29, #120] 3.49 : ffff80001045b830: ldr x1, [x19] 0.00 : ffff80001045b834: eor x1, x2, x1 0.00 : ffff80001045b838: cbnz x1, ffff80001045b9c0 1.04 : ffff80001045b83c: ldp x19, x20, [sp, #16] 0.00 : ffff80001045b840: ldp x29, x30, [sp], #128 0.00 : ffff80001045b844: ret : blkcg_punt_bio_submit(): : return __blkcg_punt_bio_submit(bio); 0.00 : ffff80001045b848: bl ffff80001047ce08 <__blkcg_punt_bio_submit> : submit_bio(): : if (blkcg_punt_bio_submit(bio)) 0.00 : ffff80001045b84c: tst w0, #0xff : return BLK_QC_T_NONE; 0.00 : ffff80001045b850: mov w1, #0xffffffff // #-1 : if (blkcg_punt_bio_submit(bio)) 0.00 : ffff80001045b854: b.eq ffff80001045b80c // b.none 0.00 : ffff80001045b858: b ffff80001045b824 : bio_has_data(): : bio_op(bio) != REQ_OP_DISCARD && 1.14 : ffff80001045b85c: ldr w2, [x20, #16] 0.00 : ffff80001045b860: and w1, w2, #0xff : bio->bi_iter.bi_size && 0.00 : ffff80001045b864: sub w3, w1, #0x3 : bio_op(bio) != REQ_OP_SECURE_ERASE && 0.00 : ffff80001045b868: tst w3, #0xfffffffd 1.17 : ffff80001045b86c: ccmp w1, #0x9, #0x4, ne // ne = any 0.00 : ffff80001045b870: b.eq ffff80001045b818 // b.none 0.00 : ffff80001045b874: str x21, [x29, #32] : submit_bio(): : count = bio_sectors(bio); 0.00 : ffff80001045b878: lsr w21, w0, #9 : if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) 0.00 : ffff80001045b87c: cmp w1, #0x7 1.17 : ffff80001045b880: mov w1, w21 9.22 : ffff80001045b884: b.eq ffff80001045b918 // b.none : if (op_is_write(bio_op(bio))) { 0.00 : ffff80001045b888: tbnz w2, #0, ffff80001045b8f4 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001045b88c: mrs x2, sp_el0 : task_io_account_read(): : #include : : #ifdef CONFIG_TASK_IO_ACCOUNTING : static inline void task_io_account_read(size_t bytes) : { : current->ioac.read_bytes += bytes; 0.00 : ffff80001045b890: ldr x4, [x2, #1928] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 8.07 : ffff80001045b894: ldr w3, [x2, #16] : task_io_account_read(): 0.00 : ffff80001045b898: add x0, x4, w0, uxtw : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001045b89c: add w3, w3, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001045b8a0: str w3, [x2, #16] : task_io_account_read(): 1.17 : ffff80001045b8a4: str x0, [x2, #1928] : count_vm_events(): : raw_cpu_add(vm_event_states.event[item], delta); : } : : static inline void count_vm_events(enum vm_event_item item, long delta) : { : this_cpu_add(vm_event_states.event[item], delta); 0.00 : ffff80001045b8a8: adrp x0, ffff800011526000 0.00 : ffff80001045b8ac: add x0, x0, #0x9b8 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 1.17 : ffff80001045b8b0: mrs x3, tpidr_el1 : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045b8b4: add x0, x0, x3 0.00 : ffff80001045b8b8: ldxr x5, [x0] 44.93 : ffff80001045b8bc: add x5, x5, x1 0.00 : ffff80001045b8c0: stxr w4, x5, [x0] 0.00 : ffff80001045b8c4: cbnz w4, ffff80001045b8b8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045b8c8: ldr x0, [x2, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045b8cc: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 5.67 : ffff80001045b8d0: str w0, [x2, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045b8d4: cbz x0, ffff80001045b910 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045b8d8: ldr x0, [x2, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045b8dc: cbz x0, ffff80001045b910 : submit_bio(): : if (unlikely(block_dump)) { 2.33 : ffff80001045b8e0: adrp x0, ffff800011b26000 0.00 : ffff80001045b8e4: ldr w0, [x0, #2560] 0.00 : ffff80001045b8e8: cbnz w0, ffff80001045b93c 4.65 : ffff80001045b8ec: ldr x21, [x29, #32] 0.00 : ffff80001045b8f0: b ffff80001045b818 : get_current(): 0.00 : ffff80001045b8f4: mrs x2, sp_el0 : __read_once_size(): 0.00 : ffff80001045b8f8: ldr w0, [x2, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff80001045b8fc: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001045b900: str w0, [x2, #16] : count_vm_events(): 0.00 : ffff80001045b904: adrp x0, ffff800011526000 0.00 : ffff80001045b908: add x0, x0, #0x9c0 0.00 : ffff80001045b90c: b ffff80001045b8b0 0.00 : ffff80001045b910: bl ffff800010cd2e78 0.00 : ffff80001045b914: b ffff80001045b8e0 : submit_bio(): : count = queue_logical_block_size(bio->bi_disk->queue) >> 9; 0.00 : ffff80001045b918: ldr x1, [x20, #8] 0.00 : ffff80001045b91c: ldr x1, [x1, #1040] : queue_logical_block_size(): : : static inline unsigned queue_logical_block_size(const struct request_queue *q) : { : int retval = 512; : : if (q && q->limits.logical_block_size) 0.00 : ffff80001045b920: cbz x1, ffff80001045b9b0 0.00 : ffff80001045b924: ldr w21, [x1, #1088] 0.00 : ffff80001045b928: cbz w21, ffff80001045b9b0 0.00 : ffff80001045b92c: lsr w1, w21, #9 0.00 : ffff80001045b930: mov x21, x1 : submit_bio(): : if (op_is_write(bio_op(bio))) { 0.00 : ffff80001045b934: tbz w2, #0, ffff80001045b88c 0.00 : ffff80001045b938: b ffff80001045b8f4 0.00 : ffff80001045b93c: stp x22, x23, [x29, #40] : printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", 0.00 : ffff80001045b940: adrp x1, ffff8000111d6000 0.00 : ffff80001045b944: stp x24, x25, [x29, #56] 0.00 : ffff80001045b948: add x1, x1, #0xa20 0.00 : ffff80001045b94c: adrp x22, ffff8000111c6000 0.00 : ffff80001045b950: add x22, x22, #0x330 : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 0.00 : ffff80001045b954: ldr w2, [x20, #16] : get_current(): 0.00 : ffff80001045b958: mrs x0, sp_el0 : submit_bio(): : (unsigned long long)bio->bi_iter.bi_sector, 0.00 : ffff80001045b95c: ldr x25, [x20, #32] : current->comm, task_pid_nr(current), 0.00 : ffff80001045b960: add x23, x0, #0x620 : printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", 0.00 : ffff80001045b964: tst x2, #0x1 : task_pid_nr(): : */ : pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns); : : static inline pid_t task_pid_nr(struct task_struct *tsk) : { : return tsk->pid; 0.00 : ffff80001045b968: ldr w24, [x0, #1128] : submit_bio(): 0.00 : ffff80001045b96c: csel x22, x22, x1, eq // eq = none 0.00 : ffff80001045b970: mov x0, x20 0.00 : ffff80001045b974: add x1, x29, #0x58 0.00 : ffff80001045b978: bl ffff800010472a70 0.00 : ffff80001045b97c: mov x5, x0 0.00 : ffff80001045b980: adrp x7, ffff8000111e9000 0.00 : ffff80001045b984: mov w6, w21 0.00 : ffff80001045b988: mov x4, x25 0.00 : ffff80001045b98c: mov x3, x22 0.00 : ffff80001045b990: mov w2, w24 0.00 : ffff80001045b994: mov x1, x23 0.00 : ffff80001045b998: add x0, x7, #0xae8 0.00 : ffff80001045b99c: bl ffff80001014a85c 0.00 : ffff80001045b9a0: ldp x21, x22, [x29, #32] 0.00 : ffff80001045b9a4: ldp x23, x24, [x29, #48] 0.00 : ffff80001045b9a8: ldr x25, [x29, #64] 0.00 : ffff80001045b9ac: b ffff80001045b818 : queue_logical_block_size(): 0.00 : ffff80001045b9b0: mov x1, #0x1 // #1 0.00 : ffff80001045b9b4: mov w21, w1 : submit_bio(): : if (op_is_write(bio_op(bio))) { 0.00 : ffff80001045b9b8: tbz w2, #0, ffff80001045b88c 0.00 : ffff80001045b9bc: b ffff80001045b8f4 0.00 : ffff80001045b9c0: stp x21, x22, [x29, #32] 0.00 : ffff80001045b9c4: stp x23, x24, [x29, #48] 0.00 : ffff80001045b9c8: str x25, [x29, #64] : } 0.00 : ffff80001045b9cc: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (27 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017c6b0 : : __arm64_get_k_vdso_data(): : * Update the vDSO data page to keep in sync with kernel timekeeping. : */ : static __always_inline : struct vdso_data *__arm64_get_k_vdso_data(void) : { : return vdso_data; 0.00 : ffff80001017c6b0: adrp x1, ffff800011915000 <__cpu_logical_map+0x680> 0.00 : ffff80001017c6b4: ldr x1, [x1, #600] : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001017c6b8: mov x7, x1 : vdso_write_begin(): : /* : * WRITE_ONCE it is required otherwise the compiler can validly tear : * updates to vd[x].seq and it is possible that the value seen by the : * reader it is inconsistent. : */ : WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1); 0.00 : ffff80001017c6bc: ldr w2, [x1] 0.00 : ffff80001017c6c0: add w2, w2, #0x1 : __write_once_size(): 0.00 : ffff80001017c6c4: str w2, [x7], #240 : vdso_write_begin(): : WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1); 0.00 : ffff80001017c6c8: ldr w2, [x1, #240] 0.00 : ffff80001017c6cc: add w2, w2, #0x1 : __write_once_size(): 0.00 : ffff80001017c6d0: str w2, [x1, #240] : vdso_write_begin(): : smp_wmb(); 0.00 : ffff80001017c6d4: dmb ishst : __arm64_get_clock_mode(): : #define __arch_get_k_vdso_data __arm64_get_k_vdso_data : : static __always_inline : int __arm64_get_clock_mode(struct timekeeper *tk) : { : u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct; 40.65 : ffff80001017c6d8: ldr x2, [x0] : __iter_div_u64_rem(): : static __always_inline u32 : __iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder) : { : u32 ret = 0; : : while (dividend >= divisor) { 0.00 : ffff80001017c6dc: mov x4, #0xc9ff // #51711 0.00 : ffff80001017c6e0: movk x4, #0x3b9a, lsl #16 : __arm64_get_clock_mode(): 3.62 : ffff80001017c6e4: ldrb w2, [x2, #36] : : return use_syscall; 0.00 : ffff80001017c6e8: eor w2, w2, #0x1 0.00 : ffff80001017c6ec: str w2, [x1, #4] : u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct; 0.00 : ffff80001017c6f0: ldr x2, [x0] 0.00 : ffff80001017c6f4: ldrb w2, [x2, #36] : return use_syscall; 0.00 : ffff80001017c6f8: eor w2, w2, #0x1 0.00 : ffff80001017c6fc: str w2, [x7, #4] : update_vsyscall(): : vdata[CS_HRES_COARSE].clock_mode = __arch_get_clock_mode(tk); : vdata[CS_RAW].clock_mode = __arch_get_clock_mode(tk); : : /* CLOCK_REALTIME also required for time() */ : vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME]; : vdso_ts->sec = tk->xtime_sec; 0.00 : ffff80001017c700: ldr x2, [x0, #112] 0.00 : ffff80001017c704: str x2, [x1, #32] : vdso_ts->nsec = tk->tkr_mono.xtime_nsec; 0.00 : ffff80001017c708: ldr x2, [x0, #32] 0.00 : ffff80001017c70c: str x2, [x1, #40] : : /* CLOCK_REALTIME_COARSE */ : vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME_COARSE]; : vdso_ts->sec = tk->xtime_sec; 0.00 : ffff80001017c710: ldr x2, [x0, #112] 0.00 : ffff80001017c714: str x2, [x1, #112] : vdso_ts->nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; 0.00 : ffff80001017c718: ldr w3, [x0, #28] 0.00 : ffff80001017c71c: ldr x2, [x0, #32] 0.00 : ffff80001017c720: lsr x2, x2, x3 0.00 : ffff80001017c724: str x2, [x1, #120] : : /* CLOCK_MONOTONIC_COARSE */ : vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC_COARSE]; : vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; 0.00 : ffff80001017c728: ldr x5, [x0, #112] 0.00 : ffff80001017c72c: ldr x2, [x0, #128] 0.00 : ffff80001017c730: add x5, x5, x2 0.00 : ffff80001017c734: str x5, [x1, #128] : nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; 0.00 : ffff80001017c738: ldr w6, [x0, #28] 0.00 : ffff80001017c73c: ldr x2, [x0, #32] : nsec = nsec + tk->wall_to_monotonic.tv_nsec; 0.00 : ffff80001017c740: ldr x3, [x0, #136] : nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; 0.00 : ffff80001017c744: lsr x2, x2, x6 : nsec = nsec + tk->wall_to_monotonic.tv_nsec; 0.00 : ffff80001017c748: add x2, x2, x3 : __iter_div_u64_rem(): 0.00 : ffff80001017c74c: cmp x2, x4 0.00 : ffff80001017c750: b.ls ffff80001017c774 // b.plast : /* The following asm() prevents the compiler from : optimising this loop into a modulo operation. */ : asm("" : "+rm"(dividend)); : : dividend -= divisor; 0.00 : ffff80001017c754: mov x6, #0xffffffffffff3600 // #-51712 : u32 ret = 0; 0.00 : ffff80001017c758: mov w3, #0x0 // #0 : dividend -= divisor; 0.00 : ffff80001017c75c: movk x6, #0xc465, lsl #16 0.00 : ffff80001017c760: add x2, x2, x6 : ret++; 0.00 : ffff80001017c764: add w3, w3, #0x1 : while (dividend >= divisor) { 0.00 : ffff80001017c768: cmp x2, x4 0.00 : ffff80001017c76c: b.hi ffff80001017c760 // b.pmore 0.00 : ffff80001017c770: add x5, x5, w3, uxtw : update_vsyscall(): : : /* : * Read without the seqlock held by clock_getres(). : * Note: No need to have a second copy. : */ : WRITE_ONCE(vdata[CS_HRES_COARSE].hrtimer_res, hrtimer_resolution); 0.00 : ffff80001017c774: adrp x3, ffff80001190a000 <__per_cpu_offset+0x6d8> : __iter_div_u64_rem(): : } : : *remainder = dividend; 0.00 : ffff80001017c778: stp x5, x2, [x1, #128] : update_vdso_data(): : while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { 0.00 : ffff80001017c77c: mov x6, #0xca00 // #51712 : update_vsyscall(): : WRITE_ONCE(vdata[CS_HRES_COARSE].hrtimer_res, hrtimer_resolution); 0.00 : ffff80001017c780: ldr w2, [x3, #700] : update_vdso_data(): : while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { 0.00 : ffff80001017c784: movk x6, #0x3b9a, lsl #16 : __write_once_size(): 0.00 : ffff80001017c788: str w2, [x1, #232] : update_vdso_data(): : vdata[CS_HRES_COARSE].cycle_last = tk->tkr_mono.cycle_last; 0.00 : ffff80001017c78c: ldr x2, [x0, #16] 0.00 : ffff80001017c790: str x2, [x1, #8] : vdata[CS_HRES_COARSE].mask = tk->tkr_mono.mask; 0.00 : ffff80001017c794: ldr x2, [x0, #8] 0.00 : ffff80001017c798: str x2, [x1, #16] : vdata[CS_HRES_COARSE].mult = tk->tkr_mono.mult; 0.00 : ffff80001017c79c: ldr w2, [x0, #24] 0.00 : ffff80001017c7a0: str w2, [x1, #24] : vdata[CS_HRES_COARSE].shift = tk->tkr_mono.shift; 0.00 : ffff80001017c7a4: ldr w2, [x0, #28] 0.00 : ffff80001017c7a8: str w2, [x1, #28] : vdata[CS_RAW].cycle_last = tk->tkr_raw.cycle_last; 0.00 : ffff80001017c7ac: ldr x2, [x0, #72] 0.00 : ffff80001017c7b0: str x2, [x7, #8] : vdata[CS_RAW].mask = tk->tkr_raw.mask; 0.00 : ffff80001017c7b4: ldr x2, [x0, #64] 0.00 : ffff80001017c7b8: str x2, [x7, #16] : vdata[CS_RAW].mult = tk->tkr_raw.mult; 0.00 : ffff80001017c7bc: ldr w2, [x0, #80] 0.00 : ffff80001017c7c0: str w2, [x7, #24] : vdata[CS_RAW].shift = tk->tkr_raw.shift; 0.00 : ffff80001017c7c4: ldr w2, [x0, #84] 0.00 : ffff80001017c7c8: str w2, [x7, #28] : vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; 0.00 : ffff80001017c7cc: ldr x2, [x0, #112] 0.00 : ffff80001017c7d0: ldr x3, [x0, #128] 0.00 : ffff80001017c7d4: add x2, x2, x3 0.00 : ffff80001017c7d8: str x2, [x1, #48] : nsec += ((u64)tk->wall_to_monotonic.tv_nsec << tk->tkr_mono.shift); 3.63 : ffff80001017c7dc: ldr w3, [x0, #28] 0.00 : ffff80001017c7e0: ldr x4, [x0, #136] : nsec = tk->tkr_mono.xtime_nsec; 0.00 : ffff80001017c7e4: ldr x5, [x0, #32] : nsec += ((u64)tk->wall_to_monotonic.tv_nsec << tk->tkr_mono.shift); 0.00 : ffff80001017c7e8: lsl x4, x4, x3 0.00 : ffff80001017c7ec: add x4, x4, x5 : while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { 0.00 : ffff80001017c7f0: lsl x3, x6, x3 0.00 : ffff80001017c7f4: cmp x4, x3 0.00 : ffff80001017c7f8: b.cc ffff80001017c824 // b.lo, b.ul, b.last 0.00 : ffff80001017c7fc: add x2, x2, #0x1 0.00 : ffff80001017c800: b ffff80001017c808 0.00 : ffff80001017c804: mov x2, x5 : vdso_ts->sec++; 0.00 : ffff80001017c808: str x2, [x1, #48] : nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift); 0.00 : ffff80001017c80c: sub x4, x4, x3 0.00 : ffff80001017c810: add x5, x2, #0x1 : while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { 0.00 : ffff80001017c814: ldr w3, [x0, #28] 0.00 : ffff80001017c818: lsl x3, x6, x3 0.00 : ffff80001017c81c: cmp x3, x4 0.00 : ffff80001017c820: b.ls ffff80001017c804 // b.plast : vdso_ts->nsec = nsec; 0.00 : ffff80001017c824: str x4, [x1, #56] : while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { 0.00 : ffff80001017c828: mov x6, #0xca00 // #51712 0.00 : ffff80001017c82c: movk x6, #0x3b9a, lsl #16 : nsec += (u64)tk->monotonic_to_boot.tv_nsec << tk->tkr_mono.shift; 0.00 : ffff80001017c830: ldp x3, x5, [x0, #200] 0.00 : ffff80001017c834: ldr w8, [x0, #28] : sec += tk->monotonic_to_boot.tv_sec; 0.00 : ffff80001017c838: add x2, x2, x3 : vdso_ts->sec = sec; 0.00 : ffff80001017c83c: str x2, [x1, #144] : while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { 0.00 : ffff80001017c840: ldr w3, [x0, #28] : nsec += (u64)tk->monotonic_to_boot.tv_nsec << tk->tkr_mono.shift; 0.00 : ffff80001017c844: lsl x5, x5, x8 0.00 : ffff80001017c848: add x4, x5, x4 : while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { 0.00 : ffff80001017c84c: lsl x3, x6, x3 0.00 : ffff80001017c850: cmp x4, x3 0.00 : ffff80001017c854: b.cc ffff80001017c87c // b.lo, b.ul, b.last 0.00 : ffff80001017c858: mov x5, x6 0.00 : ffff80001017c85c: nop : vdso_ts->sec++; 0.00 : ffff80001017c860: add x2, x2, #0x1 0.00 : ffff80001017c864: str x2, [x1, #144] : nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift); 0.00 : ffff80001017c868: sub x4, x4, x3 : while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { 0.00 : ffff80001017c86c: ldr w3, [x0, #28] 0.00 : ffff80001017c870: lsl x3, x5, x3 0.00 : ffff80001017c874: cmp x3, x4 0.00 : ffff80001017c878: b.ls ffff80001017c860 // b.plast : vdso_ts->nsec = nsec; 0.00 : ffff80001017c87c: str x4, [x1, #152] : __arm64_update_vsyscall(): : #define __arch_get_clock_mode __arm64_get_clock_mode : : static __always_inline : void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk) : { : vdata[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK; 0.00 : ffff80001017c880: mov x3, #0xffffffffffffff // #72057594037927935 : update_vdso_data(): : vdso_ts->sec = tk->raw_sec; 0.00 : ffff80001017c884: ldr x2, [x0, #192] 0.00 : ffff80001017c888: str x2, [x1, #336] : vdso_ts->nsec = tk->tkr_raw.xtime_nsec; 0.00 : ffff80001017c88c: ldr x2, [x0, #88] 0.00 : ffff80001017c890: str x2, [x1, #344] : vdso_ts->sec = tk->xtime_sec + (s64)tk->tai_offset; 0.00 : ffff80001017c894: ldrsw x4, [x0, #168] 0.00 : ffff80001017c898: ldr x2, [x0, #112] 0.00 : ffff80001017c89c: add x2, x2, x4 0.00 : ffff80001017c8a0: str x2, [x1, #208] : vdso_ts->nsec = tk->tkr_mono.xtime_nsec; 0.00 : ffff80001017c8a4: ldr x0, [x0, #32] : __arm64_update_vsyscall(): 0.00 : ffff80001017c8a8: str x3, [x1, #16] : update_vdso_data(): 0.00 : ffff80001017c8ac: str x0, [x1, #216] : __arm64_update_vsyscall(): : vdata[CS_RAW].mask = VDSO_PRECISION_MASK; 0.00 : ffff80001017c8b0: str x3, [x7, #16] : vdso_write_end(): : } : : static __always_inline void vdso_write_end(struct vdso_data *vd) : { : smp_wmb(); 0.00 : ffff80001017c8b4: dmb ishst : /* : * WRITE_ONCE it is required otherwise the compiler can validly tear : * updates to vd[x].seq and it is possible that the value seen by the : * reader it is inconsistent. : */ : WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1); 52.11 : ffff80001017c8b8: ldr w2, [x1] : WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1); 0.00 : ffff80001017c8bc: ldr w0, [x1, #240] : WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1); 0.00 : ffff80001017c8c0: add w2, w2, #0x1 : __write_once_size(): 0.00 : ffff80001017c8c4: str w2, [x1] : vdso_write_end(): : WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1); 0.00 : ffff80001017c8c8: add w0, w0, #0x1 : __write_once_size(): 0.00 : ffff80001017c8cc: str w0, [x1, #240] : update_vsyscall(): : __arch_update_vsyscall(vdata, tk); : : vdso_write_end(vdata); : : __arch_sync_vdso_data(vdata); : } 0.00 : ffff80001017c8d0: ret Percent | Source code & Disassembly of vmlinux for cycles (72 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010458448 : : bio_set_pages_dirty(): : : /* : * bio_set_pages_dirty() will mark all the bio's pages as dirty. : */ : void bio_set_pages_dirty(struct bio *bio) : { 9.75 : ffff800010458448: stp x29, x30, [sp, #-64]! 0.00 : ffff80001045844c: mov x29, sp 30.45 : ffff800010458450: stp x19, x20, [sp, #16] : bvec_init_iter_all(): : .bi_bvec_done = 0, \ : } : : static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all) : { : iter_all->done = 0; 0.00 : ffff800010458454: mov w20, #0x0 // #0 : bio_set_pages_dirty(): 2.79 : ffff800010458458: stp x21, x22, [sp, #32] 0.00 : ffff80001045845c: mov x22, x0 2.79 : ffff800010458460: str x23, [sp, #48] : bvec_init_iter_all(): : iter_all->idx = 0; 0.00 : ffff800010458464: mov w21, #0x0 // #0 0.00 : ffff800010458468: mov w23, #0x1000 // #4096 4.23 : ffff80001045846c: ldrh w4, [x0, #96] : bio_next_segment(): : struct bvec_iter_all *iter) : { : if (iter->idx >= bio->bi_vcnt) : return false; : : bvec_advance(&bio->bi_io_vec[iter->idx], iter); 0.00 : ffff800010458470: sbfiz x0, x21, #4, #32 : if (iter->idx >= bio->bi_vcnt) 0.00 : ffff800010458474: cmp w4, w21 0.00 : ffff800010458478: b.le ffff8000104584f4 : bvec_advance(&bio->bi_io_vec[iter->idx], iter); 0.00 : ffff80001045847c: ldr x3, [x22, #104] : bvec_advance(): : struct bvec_iter_all *iter_all) : { : struct bio_vec *bv = &iter_all->bv; : : if (iter_all->done) { : bv->bv_page++; 5.44 : ffff800010458480: add x19, x19, #0x40 0.00 : ffff800010458484: mov w1, #0x1000 // #4096 : bio_next_segment(): 0.00 : ffff800010458488: add x2, x3, x0 : bvec_advance(): : if (iter_all->done) { 0.00 : ffff80001045848c: cbnz w20, ffff8000104584a8 : bv->bv_offset = 0; : } else { : bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); 0.00 : ffff800010458490: ldr x19, [x3, x0] 18.15 : ffff800010458494: ldr w0, [x2, #12] : bv->bv_offset = bvec->bv_offset & ~PAGE_MASK; 0.00 : ffff800010458498: and w1, w0, #0xfff : bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT); 0.00 : ffff80001045849c: lsr w0, w0, #12 0.00 : ffff8000104584a0: sub w1, w23, w1 0.00 : ffff8000104584a4: add x19, x19, x0, lsl #6 : } : bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset, 0.00 : ffff8000104584a8: ldr w2, [x2, #8] 0.00 : ffff8000104584ac: sub w0, w2, w20 0.00 : ffff8000104584b0: cmp w0, w1 0.00 : ffff8000104584b4: csel w0, w0, w1, ls // ls = plast : bvec->bv_len - iter_all->done); : iter_all->done += bv->bv_len; 5.61 : ffff8000104584b8: add w20, w20, w0 : : if (iter_all->done == bvec->bv_len) { 0.00 : ffff8000104584bc: cmp w2, w20 0.00 : ffff8000104584c0: b.ne ffff8000104584cc // b.any : iter_all->idx++; 0.00 : ffff8000104584c4: add w21, w21, #0x1 : iter_all->done = 0; 0.00 : ffff8000104584c8: mov w20, #0x0 // #0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 13.84 : ffff8000104584cc: ldr x0, [x19] : PageCompound(): : return READ_ONCE(page->compound_head) & 1; : } : : static __always_inline int PageCompound(struct page *page) : { : return test_bit(PG_head, &page->flags) || PageTail(page); 0.00 : ffff8000104584d0: tbnz w0, #16, ffff800010458470 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.80 : ffff8000104584d4: ldr x0, [x19, #8] : PageCompound(): 0.00 : ffff8000104584d8: tbnz w0, #0, ffff800010458470 : bio_set_pages_dirty(): : struct bio_vec *bvec; : struct bvec_iter_all iter_all; : : bio_for_each_segment_all(bvec, bio, iter_all) { : if (!PageCompound(bvec->bv_page)) : set_page_dirty_lock(bvec->bv_page); 1.39 : ffff8000104584dc: mov x0, x19 0.00 : ffff8000104584e0: bl ffff8000101dc538 0.00 : ffff8000104584e4: ldrh w4, [x22, #96] : bio_next_segment(): 0.00 : ffff8000104584e8: sbfiz x0, x21, #4, #32 : if (iter->idx >= bio->bi_vcnt) 0.00 : ffff8000104584ec: cmp w4, w21 0.00 : ffff8000104584f0: b.gt ffff80001045847c : bio_set_pages_dirty(): : } : } 0.00 : ffff8000104584f4: ldp x19, x20, [sp, #16] 2.75 : ffff8000104584f8: ldp x21, x22, [sp, #32] 0.00 : ffff8000104584fc: ldr x23, [sp, #48] 0.00 : ffff800010458500: ldp x29, x30, [sp], #64 0.00 : ffff800010458504: ret Percent | Source code & Disassembly of vmlinux for cycles (67 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010488720 : : iov_iter_revert(): : } : EXPORT_SYMBOL(iov_iter_advance); : : void iov_iter_revert(struct iov_iter *i, size_t unroll) : { : if (!unroll) 16.64 : ffff800010488720: cbz x1, ffff8000104887c8 : { 3.06 : ffff800010488724: stp x29, x30, [sp, #-64]! : return; : if (WARN_ON(unroll > MAX_RW_COUNT)) 0.00 : ffff800010488728: mov x2, #0x7ffff000 // #2147479552 0.00 : ffff80001048872c: cmp x1, x2 : { 0.00 : ffff800010488730: mov x29, sp : if (WARN_ON(unroll > MAX_RW_COUNT)) 25.30 : ffff800010488734: b.hi ffff800010488800 // b.pmore : return; : i->count += unroll; 0.00 : ffff800010488738: ldr x2, [x0, #16] : iov_iter_type(): : }; : }; : : static inline enum iter_type iov_iter_type(const struct iov_iter *i) : { : return i->type & ~(READ | WRITE); 0.00 : ffff80001048873c: ldr w3, [x0] : iov_iter_revert(): 1.50 : ffff800010488740: add x2, x2, x1 1.48 : ffff800010488744: str x2, [x0, #16] : iov_iter_type(): 0.00 : ffff800010488748: and w3, w3, #0xfffffffe : iov_iter_revert(): : if (unlikely(iov_iter_is_pipe(i))) { 0.00 : ffff80001048874c: cmp w3, #0x20 0.00 : ffff800010488750: b.eq ffff800010488818 // b.none : i->iov_offset = off; : i->head = i_head; : pipe_truncate(i); : return; : } : if (unlikely(iov_iter_is_discard(i))) 17.88 : ffff800010488754: cmp w3, #0x40 0.00 : ffff800010488758: b.eq ffff8000104887c0 // b.none : return; : if (unroll <= i->iov_offset) { 5.85 : ffff80001048875c: ldr x4, [x0, #8] 0.00 : ffff800010488760: cmp x1, x4 0.00 : ffff800010488764: b.ls ffff800010488808 // b.plast 3.03 : ffff800010488768: ldp x5, x2, [x0, #24] : i->iov_offset -= unroll; : return; : } : unroll -= i->iov_offset; : if (iov_iter_is_bvec(i)) { 0.00 : ffff80001048876c: cmp w3, #0x10 : unroll -= i->iov_offset; 0.00 : ffff800010488770: sub x1, x1, x4 0.00 : ffff800010488774: add x6, x2, #0x1 11.79 : ffff800010488778: sub x3, x5, #0x10 : if (iov_iter_is_bvec(i)) { 0.00 : ffff80001048877c: b.eq ffff8000104887cc // b.none : unroll -= n; : } : } else { /* same logics for iovec and kvec */ : const struct iovec *iov = i->iov; : while (1) { : size_t n = (--iov)->iov_len; 0.00 : ffff800010488780: ldur x4, [x5, #-8] 0.00 : ffff800010488784: add x2, x2, #0x2 : i->nr_segs++; 13.47 : ffff800010488788: str x6, [x0, #32] : if (unroll <= n) { 0.00 : ffff80001048878c: cmp x1, x4 0.00 : ffff800010488790: b.ls ffff8000104887b4 // b.plast 0.00 : ffff800010488794: nop : size_t n = (--iov)->iov_len; 0.00 : ffff800010488798: sub x3, x3, #0x10 : i->iov = iov; : i->iov_offset = n - unroll; : return; : } : unroll -= n; 0.00 : ffff80001048879c: sub x1, x1, x4 : size_t n = (--iov)->iov_len; 0.00 : ffff8000104887a0: ldr x4, [x3, #8] : i->nr_segs++; 0.00 : ffff8000104887a4: str x2, [x0, #32] 0.00 : ffff8000104887a8: add x2, x2, #0x1 : if (unroll <= n) { 0.00 : ffff8000104887ac: cmp x4, x1 0.00 : ffff8000104887b0: b.cc ffff800010488798 // b.lo, b.ul, b.last : i->iov_offset = n - unroll; 0.00 : ffff8000104887b4: sub x1, x4, x1 0.00 : ffff8000104887b8: str x1, [x0, #8] : i->iov = iov; 0.00 : ffff8000104887bc: str x3, [x0, #24] : } : } : } 0.00 : ffff8000104887c0: ldp x29, x30, [sp], #64 0.00 : ffff8000104887c4: ret 0.00 : ffff8000104887c8: ret : size_t n = (--bvec)->bv_len; 0.00 : ffff8000104887cc: ldur w4, [x5, #-8] 0.00 : ffff8000104887d0: add x2, x2, #0x2 : i->nr_segs++; 0.00 : ffff8000104887d4: str x6, [x0, #32] : if (unroll <= n) { 0.00 : ffff8000104887d8: cmp x4, x1 0.00 : ffff8000104887dc: b.cs ffff8000104887b4 // b.hs, b.nlast : size_t n = (--bvec)->bv_len; 0.00 : ffff8000104887e0: sub x3, x3, #0x10 : unroll -= n; 0.00 : ffff8000104887e4: sub x1, x1, x4 : size_t n = (--bvec)->bv_len; 0.00 : ffff8000104887e8: ldr w4, [x3, #8] : i->nr_segs++; 0.00 : ffff8000104887ec: str x2, [x0, #32] 0.00 : ffff8000104887f0: add x2, x2, #0x1 : if (unroll <= n) { 0.00 : ffff8000104887f4: cmp x4, x1 0.00 : ffff8000104887f8: b.cc ffff8000104887e0 // b.lo, b.ul, b.last 0.00 : ffff8000104887fc: b ffff8000104887b4 : if (WARN_ON(unroll > MAX_RW_COUNT)) 0.00 : ffff800010488800: brk #0x800 0.00 : ffff800010488804: b ffff8000104887c0 : i->iov_offset -= unroll; 0.00 : ffff800010488808: sub x1, x4, x1 0.00 : ffff80001048880c: str x1, [x0, #8] : } 0.00 : ffff800010488810: ldp x29, x30, [sp], #64 0.00 : ffff800010488814: ret 0.00 : ffff800010488818: stp x19, x20, [x29, #16] : struct pipe_buffer *b = &pipe->bufs[i_head & p_mask]; 0.00 : ffff80001048881c: mov w7, #0x28 // #40 0.00 : ffff800010488820: stp x21, x22, [x29, #32] : b = &pipe->bufs[i_head & p_mask]; 0.00 : ffff800010488824: mov w6, w7 : struct pipe_inode_info *pipe = i->pipe; 0.00 : ffff800010488828: ldr x21, [x0, #24] : unsigned int i_head = i->head; 0.00 : ffff80001048882c: ldr w19, [x0, #32] : size_t off = i->iov_offset; 0.00 : ffff800010488830: ldr x3, [x0, #8] : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff800010488834: ldr w4, [x21, #92] : struct pipe_buffer *b = &pipe->bufs[i_head & p_mask]; 0.00 : ffff800010488838: ldr x5, [x21, #144] : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff80001048883c: sub w4, w4, #0x1 : struct pipe_buffer *b = &pipe->bufs[i_head & p_mask]; 0.00 : ffff800010488840: and w2, w4, w19 : size_t n = off - b->offset; 0.00 : ffff800010488844: umaddl x2, w2, w7, x5 0.00 : ffff800010488848: ldr w2, [x2, #8] 0.00 : ffff80001048884c: sub x2, x3, x2 : if (unroll < n) { 0.00 : ffff800010488850: cmp x1, x2 0.00 : ffff800010488854: b.cc ffff80001048888c // b.lo, b.ul, b.last : if (!unroll && i_head == i->start_head) { 0.00 : ffff800010488858: subs x1, x1, x2 0.00 : ffff80001048885c: b.ne ffff80001048886c // b.any 0.00 : ffff800010488860: ldr w2, [x0, #36] 0.00 : ffff800010488864: cmp w2, w19 0.00 : ffff800010488868: b.eq ffff800010488924 // b.none : i_head--; 0.00 : ffff80001048886c: sub w19, w19, #0x1 : b = &pipe->bufs[i_head & p_mask]; 0.00 : ffff800010488870: and w3, w4, w19 0.00 : ffff800010488874: umaddl x3, w3, w6, x5 : off = b->offset + b->len; 0.00 : ffff800010488878: ldp w2, w3, [x3, #8] 0.00 : ffff80001048887c: add w3, w2, w3 : size_t n = off - b->offset; 0.00 : ffff800010488880: sub x2, x3, w2, uxtw : if (unroll < n) { 0.00 : ffff800010488884: cmp x2, x1 0.00 : ffff800010488888: b.ls ffff800010488858 // b.plast : off -= unroll; 0.00 : ffff80001048888c: sub x1, x3, x1 : i->iov_offset = off; 0.00 : ffff800010488890: str x1, [x0, #8] : i->head = i_head; 0.00 : ffff800010488894: str w19, [x0, #32] : pipe_truncate(): : if (!pipe_empty(p_head, p_tail)) { 0.00 : ffff800010488898: ldp w20, w0, [x21, #80] 0.00 : ffff80001048889c: cmp w0, w20 0.00 : ffff8000104888a0: b.eq ffff800010488948 // b.none 0.00 : ffff8000104888a4: str x23, [x29, #48] : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff8000104888a8: ldr w22, [x21, #92] 0.00 : ffff8000104888ac: sub w22, w22, #0x1 : if (off) { 0.00 : ffff8000104888b0: cbz x1, ffff8000104888d4 : buf = &pipe->bufs[i_head & p_mask]; 0.00 : ffff8000104888b4: ldr x2, [x21, #144] 0.00 : ffff8000104888b8: and w0, w19, w22 0.00 : ffff8000104888bc: mov w3, #0x28 // #40 0.00 : ffff8000104888c0: add w19, w19, #0x1 0.00 : ffff8000104888c4: umaddl x0, w0, w3, x2 : buf->len = off - buf->offset; 0.00 : ffff8000104888c8: ldr w2, [x0, #8] 0.00 : ffff8000104888cc: sub w1, w1, w2 0.00 : ffff8000104888d0: str w1, [x0, #12] : pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]); 0.00 : ffff8000104888d4: mov w23, #0x28 // #40 : while (p_head != i_head) { 0.00 : ffff8000104888d8: cmp w19, w20 0.00 : ffff8000104888dc: b.eq ffff800010488910 // b.none : pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]); 0.00 : ffff8000104888e0: ldr x1, [x21, #144] : p_head--; 0.00 : ffff8000104888e4: sub w20, w20, #0x1 : pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]); 0.00 : ffff8000104888e8: and w2, w20, w22 : pipe_buf_release(): : struct pipe_buffer *buf) : { : const struct pipe_buf_operations *ops = buf->ops; : : buf->ops = NULL; : ops->release(pipe, buf); 0.00 : ffff8000104888ec: mov x0, x21 : pipe_truncate(): 0.00 : ffff8000104888f0: umaddl x2, w2, w23, x1 : pipe_buf_release(): 0.00 : ffff8000104888f4: mov x1, x2 : const struct pipe_buf_operations *ops = buf->ops; 0.00 : ffff8000104888f8: ldr x3, [x2, #16] : buf->ops = NULL; 0.00 : ffff8000104888fc: str xzr, [x2, #16] : ops->release(pipe, buf); 0.00 : ffff800010488900: ldr x2, [x3, #8] 0.00 : ffff800010488904: blr x2 : pipe_truncate(): : while (p_head != i_head) { 0.00 : ffff800010488908: cmp w20, w19 0.00 : ffff80001048890c: b.ne ffff8000104888e0 // b.any : pipe->head = p_head; 0.00 : ffff800010488910: str w19, [x21, #80] 0.00 : ffff800010488914: ldp x19, x20, [x29, #16] 0.00 : ffff800010488918: ldp x21, x22, [x29, #32] 0.00 : ffff80001048891c: ldr x23, [x29, #48] 0.00 : ffff800010488920: b ffff8000104887c0 : iov_iter_revert(): : i->iov_offset = off; 0.00 : ffff800010488924: str xzr, [x0, #8] : i->head = i_head; 0.00 : ffff800010488928: str w19, [x0, #32] : pipe_truncate(): : if (!pipe_empty(p_head, p_tail)) { 0.00 : ffff80001048892c: ldp w20, w0, [x21, #80] 0.00 : ffff800010488930: cmp w0, w20 0.00 : ffff800010488934: b.eq ffff800010488948 // b.none 0.00 : ffff800010488938: str x23, [x29, #48] : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff80001048893c: ldr w22, [x21, #92] 0.00 : ffff800010488940: sub w22, w22, #0x1 0.00 : ffff800010488944: b ffff8000104888d4 0.00 : ffff800010488948: ldp x19, x20, [x29, #16] 0.00 : ffff80001048894c: ldp x21, x22, [x29, #32] 0.00 : ffff800010488950: b ffff8000104887c0 Percent | Source code & Disassembly of vmlinux for cycles (75 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010714558 : : iommu_dma_map_page(): : } : : static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, : unsigned long offset, size_t size, enum dma_data_direction dir, : unsigned long attrs) : { 27.80 : ffff800010714558: stp x29, x30, [sp, #-64]! : phys_addr_t phys = page_to_phys(page) + offset; 0.00 : ffff80001071455c: adrp x6, ffff8000112f2000 : dma_info_to_prot(): : prot |= IOMMU_PRIV; 5.41 : ffff800010714560: tst x5, #0x200 : iommu_dma_map_page(): : { 0.00 : ffff800010714564: mov x29, sp 0.00 : ffff800010714568: stp x19, x20, [sp, #16] 0.00 : ffff80001071456c: stp x21, x22, [sp, #32] 0.00 : ffff800010714570: mov w21, w4 2.69 : ffff800010714574: stp x23, x24, [sp, #48] 0.00 : ffff800010714578: mov x24, x3 : phys_addr_t phys = page_to_phys(page) + offset; 1.34 : ffff80001071457c: ldr x19, [x6, #1880] : { 7.79 : ffff800010714580: mov x23, x5 : dev_is_dma_coherent(): : #elif defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ : defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ : defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) : static inline bool dev_is_dma_coherent(struct device *dev) : { : return dev->dma_coherent; 4.06 : ffff800010714584: ldrb w20, [x0, #760] : iommu_dma_map_page(): : phys_addr_t phys = page_to_phys(page) + offset; 0.00 : ffff800010714588: sub x19, x1, x19 : dev_is_dma_coherent(): 0.00 : ffff80001071458c: ubfx x20, x20, #4, #1 : iommu_dma_map_page(): 0.00 : ffff800010714590: asr x19, x19, #6 : dma_info_to_prot(): : int prot = coherent ? IOMMU_CACHE : 0; 0.00 : ffff800010714594: lsl w1, w20, #2 : iommu_dma_map_page(): : phys_addr_t phys = page_to_phys(page) + offset; 0.00 : ffff800010714598: add x19, x2, x19, lsl #12 : dma_info_to_prot(): : prot |= IOMMU_PRIV; 0.00 : ffff80001071459c: orr w2, w1, #0x20 0.00 : ffff8000107145a0: csel w1, w2, w1, ne // ne = any : switch (dir) { 0.00 : ffff8000107145a4: cmp w4, #0x1 : return prot | IOMMU_READ; 0.00 : ffff8000107145a8: orr w3, w1, #0x1 : switch (dir) { 0.00 : ffff8000107145ac: b.eq ffff8000107145c4 // b.none : return prot | IOMMU_READ | IOMMU_WRITE; 1.34 : ffff8000107145b0: orr w3, w1, #0x3 : switch (dir) { 0.00 : ffff8000107145b4: cbz w4, ffff8000107145c4 : return 0; 0.00 : ffff8000107145b8: orr w1, w1, #0x2 0.00 : ffff8000107145bc: cmp w4, #0x2 0.00 : ffff8000107145c0: csel w3, w1, wzr, eq // eq = none : iommu_dma_map_page(): : bool coherent = dev_is_dma_coherent(dev); : int prot = dma_info_to_prot(dir, coherent, attrs); : dma_addr_t dma_handle; : : dma_handle = __iommu_dma_map(dev, phys, size, prot, dma_get_mask(dev)); 4.04 : ffff8000107145c4: ldr x1, [x0, #584] : dma_get_mask(): : : static inline u64 dma_get_mask(struct device *dev) : { : if (dev->dma_mask && *dev->dma_mask) : return *dev->dma_mask; : return DMA_BIT_MASK(32); 0.00 : ffff8000107145c8: mov x4, #0xffffffff // #4294967295 : if (dev->dma_mask && *dev->dma_mask) 0.00 : ffff8000107145cc: cbz x1, ffff8000107145e0 8.11 : ffff8000107145d0: ldr x4, [x1] : return DMA_BIT_MASK(32); 0.00 : ffff8000107145d4: mov x1, #0xffffffff // #4294967295 0.00 : ffff8000107145d8: cmp x4, #0x0 0.00 : ffff8000107145dc: csel x4, x4, x1, ne // ne = any : iommu_dma_map_page(): 13.13 : ffff8000107145e0: mov x2, x24 0.00 : ffff8000107145e4: mov x1, x19 0.00 : ffff8000107145e8: bl ffff800010714410 <__iommu_dma_map> 5.37 : ffff8000107145ec: mov x22, x0 : if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && 0.00 : ffff8000107145f0: cbnz w20, ffff800010714600 0.00 : ffff8000107145f4: tst x23, #0x20 0.00 : ffff8000107145f8: ccmn x0, #0x1, #0x4, eq // eq = none 0.00 : ffff8000107145fc: b.ne ffff800010714618 // b.any : dma_handle != DMA_MAPPING_ERROR) : arch_sync_dma_for_device(phys, size, dir); : return dma_handle; : } 4.05 : ffff800010714600: mov x0, x22 1.35 : ffff800010714604: ldp x19, x20, [sp, #16] 12.17 : ffff800010714608: ldp x21, x22, [sp, #32] 1.35 : ffff80001071460c: ldp x23, x24, [sp, #48] 0.00 : ffff800010714610: ldp x29, x30, [sp], #64 0.00 : ffff800010714614: ret : arch_sync_dma_for_device(phys, size, dir); 0.00 : ffff800010714618: mov w2, w21 0.00 : ffff80001071461c: mov x1, x24 0.00 : ffff800010714620: mov x0, x19 0.00 : ffff800010714624: bl ffff8000100a1c50 : } 0.00 : ffff800010714628: mov x0, x22 0.00 : ffff80001071462c: ldp x19, x20, [sp, #16] 0.00 : ffff800010714630: ldp x21, x22, [sp, #32] 0.00 : ffff800010714634: ldp x23, x24, [sp, #48] 0.00 : ffff800010714638: ldp x29, x30, [sp], #64 0.00 : ffff80001071463c: ret Percent | Source code & Disassembly of vmlinux for cycles (70 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101dc410 : : set_page_dirty(): : * : * If the mapping doesn't provide a set_page_dirty a_op, then : * just fall through and assume that it wants buffer_heads. : */ : int set_page_dirty(struct page *page) : { 13.94 : ffff8000101dc410: stp x29, x30, [sp, #-48]! 0.00 : ffff8000101dc414: mov x29, sp 8.64 : ffff8000101dc418: stp x19, x20, [sp, #16] 0.00 : ffff8000101dc41c: adrp x20, ffff800011909000 0.00 : ffff8000101dc420: add x1, x20, #0x908 0.00 : ffff8000101dc424: mov x19, x0 0.00 : ffff8000101dc428: ldr x2, [x1] 2.88 : ffff8000101dc42c: str x2, [x29, #40] 0.00 : ffff8000101dc430: mov x2, #0x0 // #0 : struct address_space *mapping = page_mapping(page); 0.00 : ffff8000101dc434: bl ffff8000101f2c38 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101dc438: ldr x1, [x19, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff8000101dc43c: sub x2, x1, #0x1 1.34 : ffff8000101dc440: tst x1, #0x1 0.00 : ffff8000101dc444: csel x19, x2, x19, ne // ne = any : __read_once_size(): 4.30 : ffff8000101dc448: ldr x1, [x19, #8] : set_page_dirty(): : : page = compound_head(page); : if (likely(mapping)) { 0.00 : ffff8000101dc44c: cbz x0, ffff8000101dc4d4 : compound_head(): 0.00 : ffff8000101dc450: tst x1, #0x1 : set_page_dirty(): : int (*spd)(struct page *) = mapping->a_ops->set_page_dirty; 0.00 : ffff8000101dc454: ldr x2, [x0, #112] : compound_head(): 0.00 : ffff8000101dc458: sub x0, x1, #0x1 0.00 : ffff8000101dc45c: csel x0, x0, x19, ne // ne = any : set_page_dirty(): 0.00 : ffff8000101dc460: ldr x1, [x2, #24] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000101dc464: ldr x0, [x0] : set_page_dirty(): : * About lru_deactivate_page, if the page is redirty, the flag : * will be reset. So no problem. but if the page is used by readahead : * it will confuse readahead and make it restart the size rampup : * process. But it's a trivial problem. : */ : if (PageReclaim(page)) 0.00 : ffff8000101dc468: tst w0, #0x40000 0.00 : ffff8000101dc46c: b.eq ffff8000101dc490 // b.none : __read_once_size(): 0.00 : ffff8000101dc470: ldr x2, [x19, #8] : compound_head(): 0.00 : ffff8000101dc474: sub x0, x2, #0x1 0.00 : ffff8000101dc478: tst x2, #0x1 0.00 : ffff8000101dc47c: csel x0, x0, x19, ne // ne = any : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101dc480: b ffff8000101dc4c8 0.00 : ffff8000101dc484: b ffff8000101dc4c8 : __lse_atomic64_andnot(): : " " #asm_op " %[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) 0.00 : ffff8000101dc488: mov x2, #0x40000 // #262144 0.00 : ffff8000101dc48c: stclr x2, [x0] : set_page_dirty(): : ClearPageReclaim(page); : #ifdef CONFIG_BLOCK : if (!spd) : spd = __set_page_dirty_buffers; 0.00 : ffff8000101dc490: cmp x1, #0x0 0.00 : ffff8000101dc494: adrp x2, ffff8000102bf000 <__block_commit_write.isra.40+0xa8> 0.00 : ffff8000101dc498: add x2, x2, #0x930 : #endif : return (*spd)(page); 0.00 : ffff8000101dc49c: mov x0, x19 : spd = __set_page_dirty_buffers; 0.00 : ffff8000101dc4a0: csel x1, x2, x1, eq // eq = none : return (*spd)(page); 0.00 : ffff8000101dc4a4: blr x1 : if (!PageDirty(page)) { : if (!TestSetPageDirty(page)) : return 1; : } : return 0; : } 2.92 : ffff8000101dc4a8: add x20, x20, #0x908 11.45 : ffff8000101dc4ac: ldr x2, [x29, #40] 22.83 : ffff8000101dc4b0: ldr x1, [x20] 0.00 : ffff8000101dc4b4: eor x1, x2, x1 0.00 : ffff8000101dc4b8: cbnz x1, ffff8000101dc534 0.00 : ffff8000101dc4bc: ldp x19, x20, [sp, #16] 7.23 : ffff8000101dc4c0: ldp x29, x30, [sp], #48 0.00 : ffff8000101dc4c4: ret : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff8000101dc4c8: mov x2, #0x40000 // #262144 0.00 : ffff8000101dc4cc: b ffff8000101deab0 0.00 : ffff8000101dc4d0: b ffff8000101dc490 : compound_head(): 5.77 : ffff8000101dc4d4: tst x1, #0x1 0.00 : ffff8000101dc4d8: sub x0, x1, #0x1 0.00 : ffff8000101dc4dc: csel x0, x0, x19, ne // ne = any : test_bit(): 0.00 : ffff8000101dc4e0: ldr x0, [x0] : set_page_dirty(): : if (!PageDirty(page)) { 0.00 : ffff8000101dc4e4: tst w0, #0x8 0.00 : ffff8000101dc4e8: b.ne ffff8000101dc504 // b.any : __read_once_size(): 0.00 : ffff8000101dc4ec: ldr x0, [x19, #8] : compound_head(): 0.00 : ffff8000101dc4f0: sub x1, x0, #0x1 0.00 : ffff8000101dc4f4: tst x0, #0x1 0.00 : ffff8000101dc4f8: csel x19, x1, x19, ne // ne = any : __read_once_size(): 0.00 : ffff8000101dc4fc: ldr x0, [x19] : test_and_set_bit(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (READ_ONCE(*p) & mask) 0.00 : ffff8000101dc500: tbz w0, #3, ffff8000101dc50c : set_page_dirty(): : return 0; 18.69 : ffff8000101dc504: mov w0, #0x0 // #0 0.00 : ffff8000101dc508: b ffff8000101dc4a8 : arch_static_branch_jump(): 0.00 : ffff8000101dc50c: b ffff8000101dc52c 0.00 : ffff8000101dc510: b ffff8000101dc52c : __lse_atomic64_fetch_or(): : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff8000101dc514: mov x0, #0x8 // #8 0.00 : ffff8000101dc518: ldsetal x0, x0, [x19] : set_page_dirty(): : if (!TestSetPageDirty(page)) 0.00 : ffff8000101dc51c: lsr x0, x0, #3 0.00 : ffff8000101dc520: eor x0, x0, #0x1 0.00 : ffff8000101dc524: and w0, w0, #0x1 0.00 : ffff8000101dc528: b ffff8000101dc4a8 : __ll_sc_atomic64_fetch_or(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff8000101dc52c: b ffff8000101deac8 0.00 : ffff8000101dc530: b ffff8000101dc51c : set_page_dirty(): : } 0.00 : ffff8000101dc534: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (62 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102dd160 <__arm64_sys_io_getevents>: : __arm64_sys_io_getevents(): : * specifies an infinite timeout. Note that the timeout pointed to by : * timeout is relative. Will fail with -ENOSYS if not implemented. : */ : #ifdef CONFIG_64BIT : : SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id, 59.74 : ffff8000102dd160: stp x29, x30, [sp, #-96]! 0.00 : ffff8000102dd164: mov x29, sp 4.92 : ffff8000102dd168: stp x19, x20, [sp, #16] 0.00 : ffff8000102dd16c: adrp x19, ffff800011909000 0.00 : ffff8000102dd170: stp x21, x22, [sp, #32] 0.00 : ffff8000102dd174: add x1, x19, #0x908 0.00 : ffff8000102dd178: str x23, [sp, #48] 0.00 : ffff8000102dd17c: ldr x2, [x1] 16.21 : ffff8000102dd180: str x2, [x29, #88] 0.00 : ffff8000102dd184: mov x2, #0x0 // #0 0.00 : ffff8000102dd188: ldr x1, [x0, #32] 0.00 : ffff8000102dd18c: ldp x20, x21, [x0] : __se_sys_io_getevents(): 0.00 : ffff8000102dd190: ldp x22, x23, [x0, #16] : __do_sys_io_getevents(): : struct __kernel_timespec __user *, timeout) : { : struct timespec64 ts; : int ret; : : if (timeout && unlikely(get_timespec64(&ts, timeout))) 0.00 : ffff8000102dd194: cbz x1, ffff8000102dd204 <__arm64_sys_io_getevents+0xa4> 0.00 : ffff8000102dd198: add x0, x29, #0x48 0.00 : ffff8000102dd19c: bl ffff800010167ba0 0.00 : ffff8000102dd1a0: cbnz w0, ffff8000102dd20c <__arm64_sys_io_getevents+0xac> : return -EFAULT; : : ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); 0.00 : ffff8000102dd1a4: add x4, x29, #0x48 0.00 : ffff8000102dd1a8: mov x3, x23 0.00 : ffff8000102dd1ac: mov x2, x22 0.00 : ffff8000102dd1b0: mov x1, x21 0.00 : ffff8000102dd1b4: mov x0, x20 0.00 : ffff8000102dd1b8: bl ffff8000102d9be0 : if (!ret && signal_pending(current)) 0.00 : ffff8000102dd1bc: cbz w0, ffff8000102dd1ec <__arm64_sys_io_getevents+0x8c> 1.50 : ffff8000102dd1c0: sxtw x0, w0 : __arm64_sys_io_getevents(): : SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id, 0.00 : ffff8000102dd1c4: add x19, x19, #0x908 1.62 : ffff8000102dd1c8: ldr x2, [x29, #88] 3.28 : ffff8000102dd1cc: ldr x1, [x19] 0.00 : ffff8000102dd1d0: eor x1, x2, x1 0.00 : ffff8000102dd1d4: cbnz x1, ffff8000102dd214 <__arm64_sys_io_getevents+0xb4> 1.62 : ffff8000102dd1d8: ldp x19, x20, [sp, #16] 6.21 : ffff8000102dd1dc: ldp x21, x22, [sp, #32] 0.00 : ffff8000102dd1e0: ldr x23, [sp, #48] 0.00 : ffff8000102dd1e4: ldp x29, x30, [sp], #96 0.00 : ffff8000102dd1e8: ret : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000102dd1ec: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000102dd1f0: ldr x1, [x0] : __do_sys_io_getevents(): : if (!ret && signal_pending(current)) 0.00 : ffff8000102dd1f4: mov x0, #0xfffffffffffffffc // #-4 0.00 : ffff8000102dd1f8: tst x1, #0x1 0.00 : ffff8000102dd1fc: csel x0, x0, xzr, ne // ne = any 0.00 : ffff8000102dd200: b ffff8000102dd1c4 <__arm64_sys_io_getevents+0x64> : ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); 4.89 : ffff8000102dd204: mov x4, #0x0 // #0 0.00 : ffff8000102dd208: b ffff8000102dd1a8 <__arm64_sys_io_getevents+0x48> : return -EFAULT; 0.00 : ffff8000102dd20c: mov x0, #0xfffffffffffffff2 // #-14 : __arm64_sys_io_getevents(): : SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id, 0.00 : ffff8000102dd210: b ffff8000102dd1c4 <__arm64_sys_io_getevents+0x64> 0.00 : ffff8000102dd214: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (65 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104632d0 : : blk_add_timer(): : * Notes: : * Each request has its own timer, and as it is added to the queue, we : * set up the timer. When the request completes, we cancel the timer. : */ : void blk_add_timer(struct request *req) : { 0.00 : ffff8000104632d0: stp x29, x30, [sp, #-48]! 0.00 : ffff8000104632d4: mov x29, sp 0.00 : ffff8000104632d8: stp x19, x20, [sp, #16] 0.00 : ffff8000104632dc: str x21, [sp, #32] : : /* : * Some LLDs, like scsi, peek at the timeout to prevent a : * command from being retried forever. : */ : if (!req->timeout) 3.09 : ffff8000104632e0: ldr w1, [x0, #216] : struct request_queue *q = req->q; 1.55 : ffff8000104632e4: ldr x19, [x0] : if (!req->timeout) 0.00 : ffff8000104632e8: cbnz w1, ffff8000104632f4 : req->timeout = q->rq_timeout; 0.00 : ffff8000104632ec: ldr w1, [x19, #264] 7.64 : ffff8000104632f0: str w1, [x0, #216] : : req->rq_flags &= ~RQF_TIMED_OUT; : : expiry = jiffies + req->timeout; 0.00 : ffff8000104632f4: adrp x21, ffff800011907000 : req->rq_flags &= ~RQF_TIMED_OUT; 0.00 : ffff8000104632f8: ldr w2, [x0, #28] : expiry = jiffies + req->timeout; 0.00 : ffff8000104632fc: ldr x3, [x21, #2432] : req->rq_flags &= ~RQF_TIMED_OUT; 4.57 : ffff800010463300: and w2, w2, #0xffdfffff 13.75 : ffff800010463304: str w2, [x0, #28] : expiry = jiffies + req->timeout; 0.00 : ffff800010463308: add x1, x3, w1, uxtw : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 1.56 : ffff80001046330c: str x1, [x0, #224] : blk_add_timer(): : /* : * If the timer isn't already pending or this timeout is earlier : * than an existing one, modify the timer. Round up to next nearest : * second. : */ : expiry = blk_rq_timeout(round_jiffies_up(expiry)); 0.00 : ffff800010463310: mov x0, x1 0.00 : ffff800010463314: bl ffff800010169590 0.00 : ffff800010463318: mov x20, x0 : blk_rq_timeout(): : maxt = round_jiffies_up(jiffies + BLK_MAX_TIMEOUT); 0.00 : ffff80001046331c: ldr x0, [x21, #2432] 1.55 : ffff800010463320: add x0, x0, #0x4e2 0.00 : ffff800010463324: bl ffff800010169590 : blk_add_timer(): : : if (!timer_pending(&q->timeout) || 0.00 : ffff800010463328: ldr x1, [x19, #928] : blk_rq_timeout(): : if (time_after(timeout, maxt)) 0.00 : ffff80001046332c: sub x2, x0, x20 0.00 : ffff800010463330: cmp x2, #0x0 0.00 : ffff800010463334: csel x0, x0, x20, lt // lt = tstop : blk_add_timer(): : if (!timer_pending(&q->timeout) || 1.53 : ffff800010463338: cbz x1, ffff800010463364 : time_before(expiry, q->timeout.expires)) { 0.00 : ffff80001046333c: ldr x1, [x19, #936] 0.00 : ffff800010463340: sub x2, x0, x1 0.00 : ffff800010463344: tbnz x2, #63, ffff800010463358 : */ : if (!timer_pending(&q->timeout) || (diff >= HZ / 2)) : mod_timer(&q->timeout, expiry); : } : : } 0.00 : ffff800010463348: ldp x19, x20, [sp, #16] 64.77 : ffff80001046334c: ldr x21, [sp, #32] 0.00 : ffff800010463350: ldp x29, x30, [sp], #48 0.00 : ffff800010463354: ret : unsigned long diff = q->timeout.expires - expiry; 0.00 : ffff800010463358: sub x1, x1, x0 : if (!timer_pending(&q->timeout) || (diff >= HZ / 2)) 0.00 : ffff80001046335c: cmp x1, #0x7c 0.00 : ffff800010463360: b.ls ffff800010463348 // b.plast : mod_timer(&q->timeout, expiry); 0.00 : ffff800010463364: mov x1, x0 0.00 : ffff800010463368: add x0, x19, #0x398 0.00 : ffff80001046336c: bl ffff80001016a270 : } 0.00 : ffff800010463370: ldr x21, [sp, #32] 0.00 : ffff800010463374: ldp x19, x20, [sp, #16] 0.00 : ffff800010463378: ldp x29, x30, [sp], #48 0.00 : ffff80001046337c: ret Percent | Source code & Disassembly of vmlinux for cycles (61 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd2f48 : : preempt_schedule_irq(): : * off of irq context. : * Note, that this is called and return with irqs disabled. This will : * protect us against recursive calling from irq. : */ : asmlinkage __visible void __sched preempt_schedule_irq(void) : { 0.00 : ffff800010cd2f48: stp x29, x30, [sp, #-32]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 3.32 : ffff800010cd2f4c: mrs x0, sp_el0 : preempt_schedule_irq(): 0.00 : ffff800010cd2f50: mov x29, sp 0.00 : ffff800010cd2f54: stp x19, x20, [sp, #16] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 1.65 : ffff800010cd2f58: ldr w0, [x0, #16] : preempt_schedule_irq(): : enum ctx_state prev_state; : : /* Catch callers which need to be fixed */ : BUG_ON(preempt_count() || !irqs_disabled()); 0.00 : ffff800010cd2f5c: cbnz w0, ffff800010cd2fc4 : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 3.17 : ffff800010cd2f60: mrs x1, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 1.69 : ffff800010cd2f64: and w0, w1, #0x80 : preempt_schedule_irq(): 0.00 : ffff800010cd2f68: cbz w0, ffff800010cd2fc4 : arch_local_irq_enable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010cd2f6c: mov x20, #0xe0 // #224 : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010cd2f70: mov x19, #0x60 // #96 : get_current(): 1.66 : ffff800010cd2f74: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff800010cd2f78: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010cd2f7c: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010cd2f80: str w0, [x1, #16] : arch_local_irq_enable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010cd2f84: msr daifclr, #0x2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 9.89 : ffff800010cd2f88: nop : preempt_schedule_irq(): : prev_state = exception_enter(); : : do { : preempt_disable(); : local_irq_enable(); : __schedule(true); 0.00 : ffff800010cd2f8c: mov w0, #0x1 // #1 0.00 : ffff800010cd2f90: bl ffff800010cd24e0 <__schedule> : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff800010cd2f94: msr daifset, #0x2 : get_current(): 14.75 : ffff800010cd2f98: mrs x0, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd2f9c: ldr w1, [x0, #16] : __preempt_count_sub(): : } : : static inline void __preempt_count_sub(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc -= val; 0.00 : ffff800010cd2fa0: sub w1, w1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 6.50 : ffff800010cd2fa4: str w1, [x0, #16] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010cd2fa8: ldr x0, [x0] : preempt_schedule_irq(): : local_irq_disable(); : sched_preempt_enable_no_resched(); : } while (need_resched()); 0.00 : ffff800010cd2fac: tbnz w0, #1, ffff800010cd2f74 : : exception_exit(prev_state); : } 0.00 : ffff800010cd2fb0: ldp x19, x20, [sp, #16] 0.00 : ffff800010cd2fb4: ldp x29, x30, [sp], #32 0.00 : ffff800010cd2fb8: ret : arch_local_irq_enable(): : pmr_sync(); 0.00 : ffff800010cd2fbc: dsb sy 57.37 : ffff800010cd2fc0: b ffff800010cd2f8c : preempt_schedule_irq(): : BUG_ON(preempt_count() || !irqs_disabled()); 0.00 : ffff800010cd2fc4: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (76 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001045b1f8 : : blk_queue_enter(): : * blk_queue_enter() - try to increase q->q_usage_counter : * @q: request queue pointer : * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PREEMPT : */ : int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) : { 8.84 : ffff80001045b1f8: stp x29, x30, [sp, #-128]! 0.00 : ffff80001045b1fc: mov x29, sp 0.00 : ffff80001045b200: stp x19, x20, [sp, #16] 0.00 : ffff80001045b204: mov x19, x0 3.88 : ffff80001045b208: stp x21, x22, [sp, #32] : const bool pm = flags & BLK_MQ_REQ_PREEMPT; 0.00 : ffff80001045b20c: and w21, w1, #0x8 : { 0.00 : ffff80001045b210: stp x23, x24, [sp, #48] 0.00 : ffff80001045b214: add x24, x0, #0x5c0 1.33 : ffff80001045b218: str x25, [sp, #64] 0.00 : ffff80001045b21c: adrp x25, ffff800011909000 0.00 : ffff80001045b220: add x0, x25, #0x908 : rcu_read_unlock(); : : if (success) : return 0; : : if (flags & BLK_MQ_REQ_NOWAIT) 0.00 : ffff80001045b224: and w23, w1, #0x1 : { 2.57 : ffff80001045b228: ldr x2, [x0] 0.00 : ffff80001045b22c: str x2, [x29, #120] 0.00 : ffff80001045b230: mov x2, #0x0 // #0 : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045b234: mov x22, #0xffffffffffffffff // #-1 : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff80001045b238: mov x20, #0x1 // #1 1.20 : ffff80001045b23c: nop : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001045b240: bl ffff80001015c1b0 <__rcu_read_lock> 5.34 : ffff80001045b244: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001045b248: ldr x0, [x19, #1480] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001045b24c: tst x0, #0x3 0.00 : ffff80001045b250: b.ne ffff80001045b434 // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001045b254: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff80001045b258: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001045b25c: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.33 : ffff80001045b260: str w2, [x1, #16] : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001045b264: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045b268: add x0, x0, x2 0.00 : ffff80001045b26c: ldxr x4, [x0] 24.00 : ffff80001045b270: add x4, x4, x20 0.00 : ffff80001045b274: stxr w3, x4, [x0] 0.00 : ffff80001045b278: cbnz w3, ffff80001045b26c : __read_once_size(): : __READ_ONCE_SIZE; 6.66 : ffff80001045b27c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045b280: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 6.47 : ffff80001045b284: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045b288: cbz x0, ffff80001045b294 : __read_once_size(): : __READ_ONCE_SIZE; 2.69 : ffff80001045b28c: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045b290: cbnz x0, ffff80001045b398 : percpu_ref_tryget_live(): : bool ret = false; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) { : this_cpu_inc(*percpu_count); 0.00 : ffff80001045b294: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff80001045b298: bl ffff800010160d20 <__rcu_read_unlock> : blk_queue_enter(): : if (pm || !blk_queue_pm_only(q)) { 0.00 : ffff80001045b29c: cbnz w21, ffff80001045b3a0 : __read_once_size(): 0.00 : ffff80001045b2a0: ldr w0, [x19, #112] : blk_queue_enter(): 0.00 : ffff80001045b2a4: cbz w0, ffff80001045b3a0 : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff80001045b2a8: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.00 : ffff80001045b2ac: ldr x0, [x19, #1480] : __ref_is_percpu(): : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff80001045b2b0: tst x0, #0x3 0.00 : ffff80001045b2b4: b.ne ffff80001045b468 // b.any : get_current(): 0.00 : ffff80001045b2b8: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff80001045b2bc: ldr w2, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff80001045b2c0: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001045b2c4: str w2, [x1, #16] : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001045b2c8: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff80001045b2cc: add x0, x0, x2 0.00 : ffff80001045b2d0: ldxr x4, [x0] 0.00 : ffff80001045b2d4: add x4, x4, x22 0.00 : ffff80001045b2d8: stxr w3, x4, [x0] 0.00 : ffff80001045b2dc: cbnz w3, ffff80001045b2d0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045b2e0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001045b2e4: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001045b2e8: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001045b2ec: cbz x0, ffff80001045b374 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045b2f0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001045b2f4: cbz x0, ffff80001045b374 : rcu_read_unlock(): : __rcu_read_unlock(); 0.00 : ffff80001045b2f8: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff80001045b2fc: bl ffff800010160d20 <__rcu_read_unlock> : blk_queue_enter(): : if (flags & BLK_MQ_REQ_NOWAIT) 0.00 : ffff80001045b300: cbnz w23, ffff80001045b384 : * we need to order reading __PERCPU_REF_DEAD flag of : * .q_usage_counter and reading .mq_freeze_depth or : * queue dying flag, otherwise the following wait may : * never return if the two reads are reordered. : */ : smp_rmb(); 0.00 : ffff80001045b304: dmb ishld : : wait_event(q->mq_freeze_wq, 0.00 : ffff80001045b308: ldr w0, [x19, #1364] 0.00 : ffff80001045b30c: cbnz w0, ffff80001045b334 0.00 : ffff80001045b310: cbnz w21, ffff80001045b33c 0.00 : ffff80001045b314: ldr x0, [x19, #216] : blk_pm_request_resume(): : #include : : #ifdef CONFIG_PM : static inline void blk_pm_request_resume(struct request_queue *q) : { : if (q->dev && (q->rpm_status == RPM_SUSPENDED || 0.00 : ffff80001045b318: cbz x0, ffff80001045b32c 0.00 : ffff80001045b31c: ldr w1, [x19, #224] 0.00 : ffff80001045b320: sub w1, w1, #0x2 0.00 : ffff80001045b324: cmp w1, #0x1 0.00 : ffff80001045b328: b.ls ffff80001045b4a0 // b.plast : __read_once_size(): 0.00 : ffff80001045b32c: ldr w0, [x19, #112] : blk_queue_enter(): 0.00 : ffff80001045b330: cbz w0, ffff80001045b33c : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001045b334: ldr x0, [x19, #104] : blk_queue_enter(): 0.00 : ffff80001045b338: tbz w0, #1, ffff80001045b3ac : test_bit(): 0.00 : ffff80001045b33c: ldr x0, [x19, #104] : blk_queue_enter(): : (!q->mq_freeze_depth && : (pm || (blk_pm_request_resume(q), : !blk_queue_pm_only(q)))) || : blk_queue_dying(q)); : if (blk_queue_dying(q)) 0.00 : ffff80001045b340: tbz w0, #1, ffff80001045b240 : return -ENODEV; 0.00 : ffff80001045b344: mov w0, #0xffffffed // #-19 : } : } 0.00 : ffff80001045b348: add x25, x25, #0x908 2.69 : ffff80001045b34c: ldr x2, [x29, #120] 13.34 : ffff80001045b350: ldr x1, [x25] 0.00 : ffff80001045b354: eor x1, x2, x1 0.00 : ffff80001045b358: cbnz x1, ffff80001045b4b8 0.00 : ffff80001045b35c: ldp x19, x20, [sp, #16] 3.89 : ffff80001045b360: ldp x21, x22, [sp, #32] 0.00 : ffff80001045b364: ldp x23, x24, [sp, #48] 0.00 : ffff80001045b368: ldr x25, [sp, #64] 1.32 : ffff80001045b36c: ldp x29, x30, [sp], #128 0.00 : ffff80001045b370: ret : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff80001045b374: bl ffff800010cd2e78 : rcu_read_unlock(): 0.00 : ffff80001045b378: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff80001045b37c: bl ffff800010160d20 <__rcu_read_unlock> : blk_queue_enter(): : if (flags & BLK_MQ_REQ_NOWAIT) 0.00 : ffff80001045b380: cbz w23, ffff80001045b304 : return -EBUSY; 0.00 : ffff80001045b384: mov w0, #0xfffffff0 // #-16 0.00 : ffff80001045b388: b ffff80001045b348 : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff80001045b38c: b ffff80001045ce40 : atomic64_try_cmpxchg(): : static inline bool : atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) : { : s64 r, o = *old; : r = atomic64_cmpxchg(v, o, new); : if (unlikely(r != o)) 0.00 : ffff80001045b390: cmp x0, x3 0.00 : ffff80001045b394: b.ne ffff80001045b4ac // b.any : rcu_read_unlock(): 10.58 : ffff80001045b398: bl ffff800010160d20 <__rcu_read_unlock> : blk_queue_enter(): : if (pm || !blk_queue_pm_only(q)) { 2.55 : ffff80001045b39c: cbz w21, ffff80001045b2a0 : rcu_read_unlock(): 1.33 : ffff80001045b3a0: bl ffff800010160d20 <__rcu_read_unlock> : blk_queue_enter(): : return 0; 0.00 : ffff80001045b3a4: mov w0, #0x0 // #0 0.00 : ffff80001045b3a8: b ffff80001045b348 0.00 : ffff80001045b3ac: str x26, [x29, #72] : wait_event(q->mq_freeze_wq, 0.00 : ffff80001045b3b0: mov w1, #0x0 // #0 0.00 : ffff80001045b3b4: add x0, x29, #0x50 0.00 : ffff80001045b3b8: add x26, x19, #0x588 0.00 : ffff80001045b3bc: bl ffff80001012fbd8 0.00 : ffff80001045b3c0: b ffff80001045b3d8 : __read_once_size(): 0.00 : ffff80001045b3c4: ldr w0, [x19, #112] : blk_queue_enter(): 0.00 : ffff80001045b3c8: cbz w0, ffff80001045b420 : test_bit(): 0.00 : ffff80001045b3cc: ldr x0, [x19, #104] : blk_queue_enter(): 0.00 : ffff80001045b3d0: tbnz w0, #1, ffff80001045b420 0.00 : ffff80001045b3d4: bl ffff800010cd2a78 0.00 : ffff80001045b3d8: mov w2, #0x2 // #2 0.00 : ffff80001045b3dc: add x1, x29, #0x50 0.00 : ffff80001045b3e0: mov x0, x26 0.00 : ffff80001045b3e4: bl ffff80001012fcd0 0.00 : ffff80001045b3e8: ldr w0, [x19, #1364] 0.00 : ffff80001045b3ec: cbnz w0, ffff80001045b3cc 0.00 : ffff80001045b3f0: cbnz w21, ffff80001045b420 0.00 : ffff80001045b3f4: ldr x0, [x19, #216] : blk_pm_request_resume(): 0.00 : ffff80001045b3f8: cbz x0, ffff80001045b3c4 0.00 : ffff80001045b3fc: ldr w1, [x19, #224] 0.00 : ffff80001045b400: sub w1, w1, #0x2 0.00 : ffff80001045b404: cmp w1, #0x1 0.00 : ffff80001045b408: b.hi ffff80001045b3c4 // b.pmore : pm_request_resume(): : return __pm_runtime_idle(dev, RPM_ASYNC); : } : : static inline int pm_request_resume(struct device *dev) : { : return __pm_runtime_resume(dev, RPM_ASYNC); 0.00 : ffff80001045b40c: mov w1, #0x1 // #1 0.00 : ffff80001045b410: bl ffff80001073fa90 <__pm_runtime_resume> : __read_once_size(): 0.00 : ffff80001045b414: ldr w0, [x19, #112] : blk_queue_enter(): 0.00 : ffff80001045b418: cbnz w0, ffff80001045b3cc 0.00 : ffff80001045b41c: nop 0.00 : ffff80001045b420: mov x0, x26 0.00 : ffff80001045b424: add x1, x29, #0x50 0.00 : ffff80001045b428: bl ffff80001012fc00 0.00 : ffff80001045b42c: ldr x26, [x29, #72] 0.00 : ffff80001045b430: b ffff80001045b33c : percpu_ref_tryget_live(): : } else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) { 0.00 : ffff80001045b434: ldr x0, [x24, #8] 0.00 : ffff80001045b438: tbnz w0, #1, ffff80001045b2f8 : __read_once_size(): 0.00 : ffff80001045b43c: ldr x3, [x19, #1472] : atomic64_fetch_add_unless(): : atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) : { : s64 c = atomic64_read(v); : : do { : if (unlikely(c == u)) 0.00 : ffff80001045b440: cbz x3, ffff80001045b2f8 : break; : } while (!atomic64_try_cmpxchg(v, &c, c + a)); 0.00 : ffff80001045b444: add x2, x3, #0x1 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001045b448: b ffff80001045b38c 0.00 : ffff80001045b44c: b ffff80001045b38c : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff80001045b450: mov x0, x24 0.00 : ffff80001045b454: mov x1, x3 0.00 : ffff80001045b458: mov x4, x1 0.00 : ffff80001045b45c: casal x4, x2, [x24] 0.00 : ffff80001045b460: mov x0, x4 0.00 : ffff80001045b464: b ffff80001045b390 : arch_static_branch_jump(): 0.00 : ffff80001045b468: b ffff80001045b494 0.00 : ffff80001045b46c: b ffff80001045b494 : __lse_atomic64_sub_return(): : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff80001045b470: mov x0, x20 0.00 : ffff80001045b474: neg x0, x0 0.00 : ffff80001045b478: ldaddal x0, x1, [x24] 0.00 : ffff80001045b47c: add x0, x0, x1 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001045b480: cbnz x0, ffff80001045b2f8 : ref->release(ref); 0.00 : ffff80001045b484: ldr x1, [x24, #16] 0.00 : ffff80001045b488: mov x0, x24 0.00 : ffff80001045b48c: blr x1 0.00 : ffff80001045b490: b ffff80001045b2f8 : __ll_sc_atomic64_sub_return(): : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff80001045b494: b ffff80001045ce60 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff80001045b498: cbnz x0, ffff80001045b2f8 0.00 : ffff80001045b49c: b ffff80001045b484 : pm_request_resume(): 0.00 : ffff80001045b4a0: mov w1, #0x1 // #1 0.00 : ffff80001045b4a4: bl ffff80001073fa90 <__pm_runtime_resume> 0.00 : ffff80001045b4a8: b ffff80001045b32c 0.00 : ffff80001045b4ac: mov x3, x0 : atomic64_fetch_add_unless(): : if (unlikely(c == u)) 0.00 : ffff80001045b4b0: cbnz x0, ffff80001045b444 0.00 : ffff80001045b4b4: b ffff80001045b2f8 0.00 : ffff80001045b4b8: str x26, [x29, #72] : blk_queue_enter(): : } 0.00 : ffff80001045b4bc: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (62 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d8930 : : aio_read(): : } : } : : static int aio_read(struct kiocb *req, const struct iocb *iocb, : bool vectored, bool compat) : { 0.00 : ffff8000102d8930: stp x29, x30, [sp, #-272]! 0.00 : ffff8000102d8934: mov x29, sp 1.51 : ffff8000102d8938: stp x19, x20, [sp, #16] 0.00 : ffff8000102d893c: adrp x19, ffff800011909000 0.00 : ffff8000102d8940: str x21, [sp, #32] 0.00 : ffff8000102d8944: add x4, x19, #0x908 1.47 : ffff8000102d8948: stp x23, x24, [sp, #48] 0.00 : ffff8000102d894c: and w24, w2, #0xff 0.00 : ffff8000102d8950: str x25, [sp, #64] 0.00 : ffff8000102d8954: mov x23, x1 0.00 : ffff8000102d8958: mov x21, x0 0.00 : ffff8000102d895c: and w25, w3, #0xff 0.00 : ffff8000102d8960: ldr x2, [x4] 0.00 : ffff8000102d8964: str x2, [x29, #264] 0.00 : ffff8000102d8968: mov x2, #0x0 // #0 : struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; 0.00 : ffff8000102d896c: add x2, x29, #0x88 0.00 : ffff8000102d8970: str x2, [x29, #88] : struct iov_iter iter; : struct file *file; : int ret; : : ret = aio_prep_rw(req, iocb); 0.00 : ffff8000102d8974: bl ffff8000102d86b8 6.28 : ffff8000102d8978: mov w20, w0 : if (ret) 0.00 : ffff8000102d897c: cbnz w0, ffff8000102d8a10 1.62 : ffff8000102d8980: str x22, [x29, #40] : return ret; : file = req->ki_filp; 6.46 : ffff8000102d8984: ldr x22, [x21] : if (unlikely(!(file->f_mode & FMODE_READ))) 0.00 : ffff8000102d8988: ldr w1, [x22, #68] 0.00 : ffff8000102d898c: tbz w1, #0, ffff8000102d8a48 : return -EBADF; : ret = -EINVAL; : if (unlikely(!file->f_op->read_iter)) 6.33 : ffff8000102d8990: ldr x1, [x22, #40] 1.65 : ffff8000102d8994: ldr x1, [x1, #32] 0.00 : ffff8000102d8998: cbz x1, ffff8000102d8a88 : return -EINVAL; : : ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter); 21.19 : ffff8000102d899c: ldp x1, x2, [x23, #24] 0.00 : ffff8000102d89a0: mov w5, w25 0.00 : ffff8000102d89a4: add x6, x29, #0x60 0.00 : ffff8000102d89a8: mov w4, w24 1.65 : ffff8000102d89ac: add x3, x29, #0x58 0.00 : ffff8000102d89b0: bl ffff8000102d88b8 0.00 : ffff8000102d89b4: mov w20, w0 : if (ret < 0) 0.00 : ffff8000102d89b8: tbnz w0, #31, ffff8000102d8a40 : return ret; : ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); 1.57 : ffff8000102d89bc: ldr x3, [x29, #112] 0.00 : ffff8000102d89c0: add x2, x21, #0x8 0.00 : ffff8000102d89c4: mov x1, x22 0.00 : ffff8000102d89c8: mov w0, #0x0 // #0 8.15 : ffff8000102d89cc: bl ffff80001027a3a8 0.00 : ffff8000102d89d0: mov w20, w0 : if (!ret) 0.00 : ffff8000102d89d4: cbnz w0, ffff8000102d8a00 : call_read_iter(): : } ____cacheline_aligned; : : static inline ssize_t call_read_iter(struct file *file, struct kiocb *kio, : struct iov_iter *iter) : { : return file->f_op->read_iter(kio, iter); 0.00 : ffff8000102d89d8: ldr x2, [x22, #40] 0.00 : ffff8000102d89dc: add x1, x29, #0x60 0.00 : ffff8000102d89e0: mov x0, x21 14.46 : ffff8000102d89e4: ldr x2, [x2, #32] 0.00 : ffff8000102d89e8: blr x2 : aio_rw_done(): : switch (ret) { 1.65 : ffff8000102d89ec: cmn x0, #0x204 0.00 : ffff8000102d89f0: b.eq ffff8000102d8a60 // b.none 0.00 : ffff8000102d89f4: b.gt ffff8000102d8a54 0.00 : ffff8000102d89f8: cmn x0, #0x211 0.00 : ffff8000102d89fc: b.ne ffff8000102d8a64 // b.any : aio_read(): : aio_rw_done(req, call_read_iter(file, req, &iter)); : kfree(iovec); 0.00 : ffff8000102d8a00: ldr x0, [x29, #88] 0.00 : ffff8000102d8a04: bl ffff800010253890 : return ret; 4.92 : ffff8000102d8a08: ldr x22, [x29, #40] 0.00 : ffff8000102d8a0c: nop : } 14.59 : ffff8000102d8a10: add x19, x19, #0x908 0.00 : ffff8000102d8a14: mov w0, w20 0.00 : ffff8000102d8a18: ldr x2, [x29, #264] 0.00 : ffff8000102d8a1c: ldr x1, [x19] 0.00 : ffff8000102d8a20: eor x1, x2, x1 0.00 : ffff8000102d8a24: cbnz x1, ffff8000102d8a94 0.00 : ffff8000102d8a28: ldp x19, x20, [sp, #16] 6.52 : ffff8000102d8a2c: ldr x21, [sp, #32] 0.00 : ffff8000102d8a30: ldp x23, x24, [sp, #48] 0.00 : ffff8000102d8a34: ldr x25, [sp, #64] 0.00 : ffff8000102d8a38: ldp x29, x30, [sp], #272 0.00 : ffff8000102d8a3c: ret 0.00 : ffff8000102d8a40: ldr x22, [x29, #40] 0.00 : ffff8000102d8a44: b ffff8000102d8a10 : return -EBADF; 0.00 : ffff8000102d8a48: mov w20, #0xfffffff7 // #-9 0.00 : ffff8000102d8a4c: ldr x22, [x29, #40] 0.00 : ffff8000102d8a50: b ffff8000102d8a10 : aio_rw_done(): : switch (ret) { 0.00 : ffff8000102d8a54: add x1, x0, #0x202 0.00 : ffff8000102d8a58: cmp x1, #0x2 0.00 : ffff8000102d8a5c: b.hi ffff8000102d8a64 // b.pmore : ret = -EINTR; 0.00 : ffff8000102d8a60: mov x0, #0xfffffffffffffffc // #-4 : req->ki_complete(req, ret, 0); 0.00 : ffff8000102d8a64: ldr x3, [x21, #16] 0.00 : ffff8000102d8a68: mov x1, x0 0.00 : ffff8000102d8a6c: mov x2, #0x0 // #0 0.00 : ffff8000102d8a70: mov x0, x21 0.00 : ffff8000102d8a74: blr x3 : aio_read(): : kfree(iovec); 0.00 : ffff8000102d8a78: ldr x0, [x29, #88] 0.00 : ffff8000102d8a7c: bl ffff800010253890 : return ret; 0.00 : ffff8000102d8a80: ldr x22, [x29, #40] 0.00 : ffff8000102d8a84: b ffff8000102d8a10 : return -EINVAL; 0.00 : ffff8000102d8a88: mov w20, #0xffffffea // #-22 0.00 : ffff8000102d8a8c: ldr x22, [x29, #40] 0.00 : ffff8000102d8a90: b ffff8000102d8a10 0.00 : ffff8000102d8a94: str x22, [x29, #40] : } 0.00 : ffff8000102d8a98: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (74 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102cd188 : : fsnotify(): : * out to all of the registered fsnotify_group. Those groups can then use the : * notification event in whatever means they feel necessary. : */ : int fsnotify(struct inode *to_tell, __u32 mask, const void *data, int data_is, : const struct qstr *file_name, u32 cookie) : { 0.00 : ffff8000102cd188: stp x29, x30, [sp, #-176]! : struct mount *mnt = NULL; : __u32 mnt_or_sb_mask = sb->s_fsnotify_mask; : int ret = 0; : __u32 test_mask = (mask & ALL_FSNOTIFY_EVENTS); : : if (data_is == FSNOTIFY_EVENT_PATH) { 0.00 : ffff8000102cd18c: cmp w3, #0x1 : { 0.00 : ffff8000102cd190: mov x29, sp 9.44 : ffff8000102cd194: stp x19, x20, [sp, #16] 0.00 : ffff8000102cd198: mov x20, x0 1.37 : ffff8000102cd19c: str x21, [sp, #32] 0.00 : ffff8000102cd1a0: adrp x0, ffff800011909000 1.36 : ffff8000102cd1a4: str x23, [sp, #48] 0.00 : ffff8000102cd1a8: add x0, x0, #0x908 6.67 : ffff8000102cd1ac: stp x27, x28, [sp, #80] : __u32 test_mask = (mask & ALL_FSNOTIFY_EVENTS); 0.00 : ffff8000102cd1b0: mov w21, #0x1007ffff // #268959743 : { 0.00 : ffff8000102cd1b4: str w5, [x29, #108] 0.00 : ffff8000102cd1b8: mov w19, w1 1.24 : ffff8000102cd1bc: ldr x6, [x0] 0.00 : ffff8000102cd1c0: str x6, [x29, #168] 0.00 : ffff8000102cd1c4: mov x6, #0x0 // #0 : struct super_block *sb = to_tell->i_sb; 0.00 : ffff8000102cd1c8: ldr x28, [x20, #40] : { 0.00 : ffff8000102cd1cc: stp x4, x2, [x29, #112] 0.00 : ffff8000102cd1d0: mov x27, #0x0 // #0 : struct fsnotify_iter_info iter_info = {}; 1.37 : ffff8000102cd1d4: stp xzr, xzr, [x29, #136] : { 0.00 : ffff8000102cd1d8: mov w23, w3 : struct fsnotify_iter_info iter_info = {}; 2.58 : ffff8000102cd1dc: stp xzr, xzr, [x29, #152] : __u32 test_mask = (mask & ALL_FSNOTIFY_EVENTS); 1.39 : ffff8000102cd1e0: and w21, w1, w21 : __u32 mnt_or_sb_mask = sb->s_fsnotify_mask; 1.36 : ffff8000102cd1e4: ldr w0, [x28, #960] : if (data_is == FSNOTIFY_EVENT_PATH) { 0.00 : ffff8000102cd1e8: b.ne ffff8000102cd1fc // b.any : mnt = real_mount(((const struct path *)data)->mnt); 1.35 : ffff8000102cd1ec: ldr x1, [x2] : real_mount(): : : #define MNT_NS_INTERNAL ERR_PTR(-EINVAL) /* distinct from any mnt_namespace */ : : static inline struct mount *real_mount(struct vfsmount *mnt) : { : return container_of(mnt, struct mount, mnt); 0.00 : ffff8000102cd1f0: sub x27, x1, #0x20 : fsnotify(): : mnt_or_sb_mask |= mnt->mnt_fsnotify_mask; 1.29 : ffff8000102cd1f4: ldr w2, [x27, #280] 0.00 : ffff8000102cd1f8: orr w0, w0, w2 : * be expensive. It protects walking the *_fsnotify_marks lists. : * However, if we do not walk the lists, we do not have to do : * SRCU because we have no references to any objects and do not : * need SRCU to keep them "alive". : */ : if (!to_tell->i_fsnotify_marks && !sb->s_fsnotify_marks && 16.33 : ffff8000102cd1fc: ldr x2, [x20, #560] : mnt_or_sb_mask = 0; 0.00 : ffff8000102cd200: tst x19, #0x8000000 0.00 : ffff8000102cd204: csel w0, w0, wzr, eq // eq = none : if (!to_tell->i_fsnotify_marks && !sb->s_fsnotify_marks && 0.00 : ffff8000102cd208: cbz x2, ffff8000102cd498 0.00 : ffff8000102cd20c: str x22, [x29, #40] : /* : * if this is a modify event we may need to clear the ignored masks : * otherwise return if neither the inode nor the vfsmount/sb care about : * this type of event. : */ : if (!(mask & FS_MODIFY) && 0.00 : ffff8000102cd210: and w22, w19, #0x2 0.00 : ffff8000102cd214: tbnz w19, #1, ffff8000102cd22c : !(test_mask & (to_tell->i_fsnotify_mask | mnt_or_sb_mask))) 1.35 : ffff8000102cd218: ldr w3, [x20, #556] : return 0; 0.00 : ffff8000102cd21c: mov w2, #0x0 // #0 : !(test_mask & (to_tell->i_fsnotify_mask | mnt_or_sb_mask))) 0.00 : ffff8000102cd220: orr w0, w0, w3 : if (!(mask & FS_MODIFY) && 0.00 : ffff8000102cd224: tst w0, w21 0.00 : ffff8000102cd228: b.eq ffff8000102cd490 // b.none 0.00 : ffff8000102cd22c: stp x24, x25, [x29, #56] : srcu_read_lock(): : */ : static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp) : { : int retval; : : retval = __srcu_read_lock(ssp); 0.00 : ffff8000102cd230: adrp x0, ffff800011b32000 0.00 : ffff8000102cd234: str x26, [x29, #72] 0.00 : ffff8000102cd238: add x0, x0, #0x9b0 0.00 : ffff8000102cd23c: bl ffff80001015b3d8 <__srcu_read_lock> : fsnotify(): : return 0; : : iter_info.srcu_idx = srcu_read_lock(&fsnotify_mark_srcu); 0.00 : ffff8000102cd240: str w0, [x29, #164] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000102cd244: ldr x2, [x20, #560] : fsnotify_first_mark(): : return hlist_entry_safe(node, struct fsnotify_mark, obj_list); 0.00 : ffff8000102cd248: mov x25, #0x0 // #0 : if (conn) 0.00 : ffff8000102cd24c: cbz x2, ffff8000102cd260 : __read_once_size(): 0.00 : ffff8000102cd250: ldr x0, [x2, #24] : fsnotify_first_mark(): : return hlist_entry_safe(node, struct fsnotify_mark, obj_list); 0.00 : ffff8000102cd254: sub x3, x0, #0x28 0.00 : ffff8000102cd258: cmp x0, #0x0 0.00 : ffff8000102cd25c: csel x25, x3, xzr, ne // ne = any : __read_once_size(): 0.00 : ffff8000102cd260: ldr x2, [x28, #968] : fsnotify_first_mark(): 0.00 : ffff8000102cd264: mov x0, #0x0 // #0 : fsnotify(): : : iter_info.marks[FSNOTIFY_OBJ_TYPE_INODE] = 0.00 : ffff8000102cd268: str x25, [x29, #136] : fsnotify_first_mark(): : if (conn) 0.00 : ffff8000102cd26c: cbz x2, ffff8000102cd280 : __read_once_size(): 0.00 : ffff8000102cd270: ldr x2, [x2, #24] : fsnotify_first_mark(): : return hlist_entry_safe(node, struct fsnotify_mark, obj_list); 0.00 : ffff8000102cd274: sub x0, x2, #0x28 0.00 : ffff8000102cd278: cmp x2, #0x0 0.00 : ffff8000102cd27c: csel x0, x0, xzr, ne // ne = any : fsnotify(): : fsnotify_first_mark(&to_tell->i_fsnotify_marks); : iter_info.marks[FSNOTIFY_OBJ_TYPE_SB] = 0.00 : ffff8000102cd280: str x0, [x29, #152] : fsnotify_first_mark(&sb->s_fsnotify_marks); : if (mnt) { 0.00 : ffff8000102cd284: cbz x27, ffff8000102cd2a8 : __read_once_size(): 0.00 : ffff8000102cd288: ldr x1, [x27, #272] : fsnotify_first_mark(): : return hlist_entry_safe(node, struct fsnotify_mark, obj_list); 0.00 : ffff8000102cd28c: mov x0, #0x0 // #0 : if (conn) 0.00 : ffff8000102cd290: cbz x1, ffff8000102cd2a4 : __read_once_size(): 0.00 : ffff8000102cd294: ldr x1, [x1, #24] : fsnotify_first_mark(): : return hlist_entry_safe(node, struct fsnotify_mark, obj_list); 0.00 : ffff8000102cd298: sub x0, x1, #0x28 0.00 : ffff8000102cd29c: cmp x1, #0x0 0.00 : ffff8000102cd2a0: csel x0, x0, xzr, ne // ne = any : fsnotify(): : iter_info.marks[FSNOTIFY_OBJ_TYPE_VFSMOUNT] = 0.00 : ffff8000102cd2a4: str x0, [x29, #144] 0.00 : ffff8000102cd2a8: add x27, x29, #0x88 : fsnotify_iter_set_report_type(): : } : : static inline void fsnotify_iter_set_report_type( : struct fsnotify_iter_info *iter_info, int type) : { : iter_info->report_mask |= (1U << type); 0.00 : ffff8000102cd2ac: mov w28, #0x1 // #1 : fsnotify_next_mark(): : return hlist_entry_safe(node, struct fsnotify_mark, obj_list); 0.00 : ffff8000102cd2b0: mov x24, #0x0 // #0 : fsnotify_iter_select_report_types(): : struct fsnotify_group *max_prio_group = NULL; 0.00 : ffff8000102cd2b4: mov x26, #0x0 // #0 : if (mark && 0.00 : ffff8000102cd2b8: cbz x25, ffff8000102cd2d4 : fsnotify_compare_groups(max_prio_group, mark->group) > 0) 0.00 : ffff8000102cd2bc: ldr x1, [x25, #8] 0.00 : ffff8000102cd2c0: mov x0, x26 0.00 : ffff8000102cd2c4: bl ffff8000102ceba0 : if (mark && 0.00 : ffff8000102cd2c8: cmp w0, #0x0 0.00 : ffff8000102cd2cc: b.le ffff8000102cd2d4 : max_prio_group = mark->group; 0.00 : ffff8000102cd2d0: ldr x26, [x25, #8] 0.00 : ffff8000102cd2d4: add x24, x24, #0x8 : fsnotify_foreach_obj_type(type) { 0.00 : ffff8000102cd2d8: cmp x24, #0x18 0.00 : ffff8000102cd2dc: b.eq ffff8000102cd2e8 // b.none 0.00 : ffff8000102cd2e0: ldr x25, [x27, x24] 0.00 : ffff8000102cd2e4: b ffff8000102cd2b8 : if (!max_prio_group) 0.00 : ffff8000102cd2e8: cbz x26, ffff8000102cd42c : iter_info->report_mask = 0; 0.00 : ffff8000102cd2ec: mov x25, #0x0 // #0 0.00 : ffff8000102cd2f0: str wzr, [x29, #160] : mark = iter_info->marks[type]; 0.00 : ffff8000102cd2f4: ldr x0, [x27, x25, lsl #3] : if (mark && 0.00 : ffff8000102cd2f8: cbz x0, ffff8000102cd31c : fsnotify_compare_groups(max_prio_group, mark->group) == 0) 0.00 : ffff8000102cd2fc: ldr x1, [x0, #8] 0.00 : ffff8000102cd300: mov x0, x26 0.00 : ffff8000102cd304: bl ffff8000102ceba0 : if (mark && 0.00 : ffff8000102cd308: cbnz w0, ffff8000102cd31c : fsnotify_iter_set_report_type(): 0.00 : ffff8000102cd30c: ldr w0, [x29, #160] 0.00 : ffff8000102cd310: lsl w3, w28, w25 0.00 : ffff8000102cd314: orr w3, w0, w3 0.00 : ffff8000102cd318: str w3, [x29, #160] 0.00 : ffff8000102cd31c: add x25, x25, #0x1 : fsnotify_iter_select_report_types(): : fsnotify_foreach_obj_type(type) { 0.00 : ffff8000102cd320: cmp x25, #0x3 0.00 : ffff8000102cd324: b.ne ffff8000102cd2f4 // b.any : return iter_info->report_mask; 0.00 : ffff8000102cd328: ldr w2, [x29, #160] : fsnotify(): : /* : * We need to merge inode/vfsmount/sb mark lists so that e.g. inode mark : * ignore masks are properly reflected for mount/sb mark notifications. : * That's why this traversal is so complicated... : */ : while (fsnotify_iter_select_report_types(&iter_info)) { 0.00 : ffff8000102cd32c: cbz w2, ffff8000102cd42c : send_to_group(): : if (mask & FS_MODIFY) { 0.00 : ffff8000102cd330: cbz w22, ffff8000102cd364 0.00 : ffff8000102cd334: mov x0, #0x0 // #0 : fsnotify_iter_should_report_type(): : return (iter_info->report_mask & (1U << type)); 0.00 : ffff8000102cd338: lsl w1, w28, w0 : send_to_group(): : if (!fsnotify_iter_should_report_type(iter_info, type)) 0.00 : ffff8000102cd33c: tst w1, w2 0.00 : ffff8000102cd340: b.eq ffff8000102cd358 // b.none : mark = iter_info->marks[type]; 0.00 : ffff8000102cd344: ldr x1, [x27, x0, lsl #3] : if (mark && 0.00 : ffff8000102cd348: cbz x1, ffff8000102cd358 : !(mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY)) 0.00 : ffff8000102cd34c: ldr w3, [x1, #68] : if (mark && 0.00 : ffff8000102cd350: tbnz w3, #0, ffff8000102cd358 : mark->ignored_mask = 0; 0.00 : ffff8000102cd354: str wzr, [x1, #64] 0.00 : ffff8000102cd358: add x0, x0, #0x1 : fsnotify_foreach_obj_type(type) { 0.00 : ffff8000102cd35c: cmp x0, #0x3 0.00 : ffff8000102cd360: b.ne ffff8000102cd338 // b.any : if (mask & FS_MODIFY) { 0.00 : ffff8000102cd364: mov x1, #0x0 // #0 0.00 : ffff8000102cd368: mov x0, #0x0 // #0 0.00 : ffff8000102cd36c: mov w5, #0x0 // #0 0.00 : ffff8000102cd370: mov w4, #0x0 // #0 : fsnotify_iter_should_report_type(): 0.00 : ffff8000102cd374: lsl w3, w28, w1 : send_to_group(): : if (!fsnotify_iter_should_report_type(iter_info, type)) 0.00 : ffff8000102cd378: tst w3, w2 0.00 : ffff8000102cd37c: b.eq ffff8000102cd39c // b.none : mark = iter_info->marks[type]; 0.00 : ffff8000102cd380: ldr x3, [x27, x1, lsl #3] : if (mark) { 0.00 : ffff8000102cd384: cbz x3, ffff8000102cd39c : marks_mask |= mark->mask; 0.00 : ffff8000102cd388: ldr w7, [x3] : marks_ignored_mask |= mark->ignored_mask; 0.00 : ffff8000102cd38c: ldr w6, [x3, #64] : group = mark->group; 0.00 : ffff8000102cd390: ldr x0, [x3, #8] : marks_mask |= mark->mask; 0.00 : ffff8000102cd394: orr w4, w4, w7 : marks_ignored_mask |= mark->ignored_mask; 0.00 : ffff8000102cd398: orr w5, w5, w6 0.00 : ffff8000102cd39c: add x1, x1, #0x1 : fsnotify_foreach_obj_type(type) { 0.00 : ffff8000102cd3a0: cmp x1, #0x3 0.00 : ffff8000102cd3a4: b.ne ffff8000102cd374 // b.any : if (!(test_mask & marks_mask & ~marks_ignored_mask)) 0.00 : ffff8000102cd3a8: and w4, w21, w4 0.00 : ffff8000102cd3ac: bics wzr, w4, w5 0.00 : ffff8000102cd3b0: b.eq ffff8000102cd3ec // b.none : return group->ops->handle_event(group, to_tell, mask, data, data_is, 0.00 : ffff8000102cd3b4: ldr x1, [x0] 0.00 : ffff8000102cd3b8: mov w2, w19 0.00 : ffff8000102cd3bc: ldr w6, [x29, #108] 0.00 : ffff8000102cd3c0: mov x7, x27 0.00 : ffff8000102cd3c4: ldp x5, x3, [x29, #112] 0.00 : ffff8000102cd3c8: mov w4, w23 0.00 : ffff8000102cd3cc: ldr x10, [x1] 0.00 : ffff8000102cd3d0: mov x1, x20 0.00 : ffff8000102cd3d4: blr x10 0.00 : ffff8000102cd3d8: mov w2, w0 : fsnotify(): : ret = send_to_group(to_tell, mask, data, data_is, cookie, : file_name, &iter_info); : : if (ret && (mask & ALL_FSNOTIFY_PERM_EVENTS)) 0.00 : ffff8000102cd3dc: cbz w0, ffff8000102cd3e8 0.00 : ffff8000102cd3e0: tst w19, #0x70000 0.00 : ffff8000102cd3e4: b.ne ffff8000102cd430 // b.any 0.00 : ffff8000102cd3e8: ldr w2, [x29, #160] : send_to_group(): : if (mask & FS_MODIFY) { 0.00 : ffff8000102cd3ec: mov x0, #0x0 // #0 : fsnotify_iter_should_report_type(): 0.00 : ffff8000102cd3f0: lsl w1, w28, w0 : fsnotify_iter_next(): : if (fsnotify_iter_should_report_type(iter_info, type)) 0.00 : ffff8000102cd3f4: tst w1, w2 0.00 : ffff8000102cd3f8: b.eq ffff8000102cd418 // b.none : fsnotify_next_mark(iter_info->marks[type]); 0.00 : ffff8000102cd3fc: ldr x1, [x27, x0, lsl #3] : fsnotify_next_mark(): : if (mark) 0.00 : ffff8000102cd400: cbz x1, ffff8000102cd414 : __read_once_size(): 0.00 : ffff8000102cd404: ldr x3, [x1, #40] : fsnotify_next_mark(): : return hlist_entry_safe(node, struct fsnotify_mark, obj_list); 0.00 : ffff8000102cd408: sub x1, x3, #0x28 0.00 : ffff8000102cd40c: cmp x3, #0x0 0.00 : ffff8000102cd410: csel x1, x1, xzr, ne // ne = any : fsnotify_iter_next(): : iter_info->marks[type] = 0.00 : ffff8000102cd414: str x1, [x27, x0, lsl #3] 0.00 : ffff8000102cd418: add x0, x0, #0x1 : fsnotify_foreach_obj_type(type) { 0.00 : ffff8000102cd41c: cmp x0, #0x3 0.00 : ffff8000102cd420: b.ne ffff8000102cd3f0 // b.any 0.00 : ffff8000102cd424: ldr x25, [x29, #136] 0.00 : ffff8000102cd428: b ffff8000102cd2b0 : fsnotify(): : goto out; : : fsnotify_iter_next(&iter_info); : } : ret = 0; 0.00 : ffff8000102cd42c: mov w2, #0x0 // #0 : out: : srcu_read_unlock(&fsnotify_mark_srcu, iter_info.srcu_idx); 0.00 : ffff8000102cd430: ldr w1, [x29, #164] : srcu_read_unlock(): : * Exit an SRCU read-side critical section. : */ : static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) : __releases(ssp) : { : WARN_ON_ONCE(idx & ~0x1); 0.00 : ffff8000102cd434: tst w1, #0xfffffffe 0.00 : ffff8000102cd438: b.ne ffff8000102cd4b0 // b.any 0.00 : ffff8000102cd43c: str w2, [x29, #120] : rcu_lock_release(&(ssp)->dep_map); : __srcu_read_unlock(ssp, idx); 0.00 : ffff8000102cd440: adrp x0, ffff800011b32000 0.00 : ffff8000102cd444: add x0, x0, #0x9b0 0.00 : ffff8000102cd448: bl ffff80001015b368 <__srcu_read_unlock> 0.00 : ffff8000102cd44c: ldr w2, [x29, #120] 0.00 : ffff8000102cd450: ldr x22, [x29, #40] 0.00 : ffff8000102cd454: ldp x24, x25, [x29, #56] 0.00 : ffff8000102cd458: ldr x26, [x29, #72] : fsnotify(): : : return ret; : } 13.70 : ffff8000102cd45c: adrp x0, ffff800011909000 0.00 : ffff8000102cd460: add x27, x0, #0x908 0.00 : ffff8000102cd464: mov w0, w2 0.00 : ffff8000102cd468: ldr x2, [x29, #168] 10.66 : ffff8000102cd46c: ldr x1, [x27] 0.00 : ffff8000102cd470: eor x1, x2, x1 0.00 : ffff8000102cd474: cbnz x1, ffff8000102cd4b8 9.54 : ffff8000102cd478: ldp x19, x20, [sp, #16] 8.17 : ffff8000102cd47c: ldr x21, [sp, #32] 0.00 : ffff8000102cd480: ldr x23, [sp, #48] 0.00 : ffff8000102cd484: ldp x27, x28, [sp, #80] 1.37 : ffff8000102cd488: ldp x29, x30, [sp], #176 0.00 : ffff8000102cd48c: ret 9.47 : ffff8000102cd490: ldr x22, [x29, #40] 0.00 : ffff8000102cd494: b ffff8000102cd45c : if (!to_tell->i_fsnotify_marks && !sb->s_fsnotify_marks && 0.00 : ffff8000102cd498: ldr x2, [x28, #968] 0.00 : ffff8000102cd49c: cbnz x2, ffff8000102cd20c 0.00 : ffff8000102cd4a0: cbz x27, ffff8000102cd45c : (!mnt || !mnt->mnt_fsnotify_marks)) 0.00 : ffff8000102cd4a4: ldr x3, [x27, #272] 0.00 : ffff8000102cd4a8: cbnz x3, ffff8000102cd20c 0.00 : ffff8000102cd4ac: b ffff8000102cd45c : srcu_read_unlock(): : WARN_ON_ONCE(idx & ~0x1); 0.00 : ffff8000102cd4b0: brk #0x800 0.00 : ffff8000102cd4b4: b ffff8000102cd43c 0.00 : ffff8000102cd4b8: str x22, [x29, #40] 0.00 : ffff8000102cd4bc: stp x24, x25, [x29, #56] 0.00 : ffff8000102cd4c0: str x26, [x29, #72] : fsnotify(): : } 0.00 : ffff8000102cd4c4: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (25 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010160f40 : : rcu_idle_enter(): : * : * If you add or remove a call to rcu_idle_enter(), be sure to test with : * CONFIG_RCU_EQS_DEBUG=y. : */ : void rcu_idle_enter(void) : { 8.66 : ffff800010160f40: stp x29, x30, [sp, #-32]! : rcu_eqs_enter(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff800010160f44: adrp x2, ffff800011529000 0.00 : ffff800010160f48: add x0, x2, #0xac0 : WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); 0.00 : ffff800010160f4c: mov x1, #0x4000000000000000 // #4611686018427387904 : rcu_idle_enter(): : { 27.83 : ffff800010160f50: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010160f54: mrs x3, tpidr_el1 : rcu_eqs_enter(): : struct rcu_data *rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff800010160f58: add x0, x0, x3 : WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); 0.00 : ffff800010160f5c: ldr x3, [x0, #208] 0.00 : ffff800010160f60: cmp x3, x1 0.00 : ffff800010160f64: b.ne ffff800010160fb8 // b.any : if (rdp->dynticks_nesting != 1) { 4.15 : ffff800010160f68: ldr x1, [x0, #200] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 16.05 : ffff800010160f6c: str xzr, [x0, #208] : rcu_eqs_enter(): 0.00 : ffff800010160f70: cmp x1, #0x1 0.00 : ffff800010160f74: b.eq ffff800010160f88 // b.none : rdp->dynticks_nesting--; 0.00 : ffff800010160f78: sub x1, x1, #0x1 0.00 : ffff800010160f7c: str x1, [x0, #200] : rcu_idle_enter(): : lockdep_assert_irqs_disabled(); : rcu_eqs_enter(false); : } 0.00 : ffff800010160f80: ldp x29, x30, [sp], #32 0.00 : ffff800010160f84: ret 0.00 : ffff800010160f88: str x19, [x29, #16] : rcu_eqs_enter(): : rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff800010160f8c: add x2, x2, #0xac0 : __my_cpu_offset(): 19.91 : ffff800010160f90: mrs x19, tpidr_el1 : rcu_eqs_enter(): 0.00 : ffff800010160f94: add x19, x2, x19 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010160f98: ldr w0, [x0, #216] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010160f9c: mrs x0, sp_el0 : rcu_eqs_enter(): : rcu_preempt_deferred_qs(current); 0.00 : ffff800010160fa0: bl ffff800010160210 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 23.41 : ffff800010160fa4: str xzr, [x19, #200] : rcu_eqs_enter(): : rcu_dynticks_eqs_enter(); 0.00 : ffff800010160fa8: bl ffff80001015ded0 0.00 : ffff800010160fac: ldr x19, [x29, #16] : rcu_idle_enter(): : } 0.00 : ffff800010160fb0: ldp x29, x30, [sp], #32 0.00 : ffff800010160fb4: ret : rcu_eqs_enter(): : WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE); 0.00 : ffff800010160fb8: brk #0x800 0.00 : ffff800010160fbc: b ffff800010160f68 Percent | Source code & Disassembly of vmlinux for cycles (63 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101d1588 : : unlock_page(): : * clear the PG_locked bit and test PG_waiters at the same time fairly : * portably (architectures that do LL/SC can test any bit, while x86 can : * test the sign bit). : */ : void unlock_page(struct page *page) : { 3.26 : ffff8000101d1588: stp x29, x30, [sp, #-16]! 0.00 : ffff8000101d158c: mov x29, sp : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 15.92 : ffff8000101d1590: ldr x1, [x0, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff8000101d1594: sub x2, x1, #0x1 0.00 : ffff8000101d1598: tst x1, #0x1 0.00 : ffff8000101d159c: csel x0, x2, x0, ne // ne = any : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 3.21 : ffff8000101d15a0: b ffff8000101d15c4 0.00 : ffff8000101d15a4: b ffff8000101d15c4 : __lse_atomic64_fetch_andnot_release(): : ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \ : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) 0.00 : ffff8000101d15a8: mov x1, #0x1 // #1 0.00 : ffff8000101d15ac: ldclrl x1, x1, [x0] : unlock_page(): : BUILD_BUG_ON(PG_waiters != 7); : page = compound_head(page); : VM_BUG_ON_PAGE(!PageLocked(page), page); : if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags)) 76.04 : ffff8000101d15b0: tbz w1, #7, ffff8000101d15bc : wake_up_page_bit(page, PG_locked); 0.00 : ffff8000101d15b4: mov w1, #0x0 // #0 0.00 : ffff8000101d15b8: bl ffff8000101d1438 : } 0.00 : ffff8000101d15bc: ldp x29, x30, [sp], #16 1.58 : ffff8000101d15c0: ret : __ll_sc_atomic64_fetch_andnot_release(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff8000101d15c4: mov x2, #0x1 // #1 0.00 : ffff8000101d15c8: b ffff8000101d6c78 0.00 : ffff8000101d15cc: b ffff8000101d15b0 Percent | Source code & Disassembly of vmlinux for cycles (53 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010714410 <__iommu_dma_map>: : __iommu_dma_map(): : iommu_dma_free_iova(cookie, dma_addr, size); : } : : static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, : size_t size, int prot, u64 dma_mask) : { 1.92 : ffff800010714410: stp x29, x30, [sp, #-80]! 0.00 : ffff800010714414: mov x29, sp 30.36 : ffff800010714418: stp x19, x20, [sp, #16] 0.00 : ffff80001071441c: mov x20, x0 0.00 : ffff800010714420: stp x21, x22, [sp, #32] 0.00 : ffff800010714424: mov x22, x1 0.00 : ffff800010714428: stp x23, x24, [sp, #48] 0.00 : ffff80001071442c: mov w23, w3 0.00 : ffff800010714430: stp x25, x26, [sp, #64] 0.00 : ffff800010714434: sub x24, x2, #0x1 0.00 : ffff800010714438: mov x25, x4 : struct iommu_domain *domain = iommu_get_dma_domain(dev); 0.00 : ffff80001071443c: bl ffff800010712818 25.92 : ffff800010714440: mov x21, x0 : struct iommu_dma_cookie *cookie = domain->iova_cookie; : struct iova_domain *iovad = &cookie->iovad; : size_t iova_off = iova_offset(iovad, phys); : dma_addr_t iova; : : if (unlikely(iommu_dma_deferred_attach(dev, domain))) 0.00 : ffff800010714444: mov x0, x20 0.00 : ffff800010714448: mov x1, x21 : struct iommu_dma_cookie *cookie = domain->iova_cookie; 0.00 : ffff80001071444c: ldr x26, [x21, #64] : iova_mask(): : return __ffs(iovad->granule); : } : : static inline unsigned long iova_mask(struct iova_domain *iovad) : { : return iovad->granule - 1; 5.57 : ffff800010714450: ldr x19, [x26, #40] 0.00 : ffff800010714454: sub x19, x19, #0x1 : iova_offset(): : } : : static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova) : { : return iova & iova_mask(iovad); 0.00 : ffff800010714458: and x19, x19, x22 : __iommu_dma_map(): : if (unlikely(iommu_dma_deferred_attach(dev, domain))) 0.00 : ffff80001071445c: bl ffff800010713bb8 1.90 : ffff800010714460: cbnz w0, ffff8000107144c8 <__iommu_dma_map+0xb8> : return DMA_MAPPING_ERROR; : : size = iova_align(iovad, size + iova_off); 3.83 : ffff800010714464: ldr x0, [x26, #40] : : iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev); 0.00 : ffff800010714468: add x4, x20, #0x258 0.00 : ffff80001071446c: add x3, x20, #0x60 0.00 : ffff800010714470: mov x2, x25 : iova_align(): : } : : static inline size_t iova_align(struct iova_domain *iovad, size_t size) : { : return ALIGN(size, iovad->granule); 9.40 : ffff800010714474: add x20, x0, x19 0.00 : ffff800010714478: neg x1, x0 0.00 : ffff80001071447c: add x20, x20, x24 : __iommu_dma_map(): 0.00 : ffff800010714480: mov x0, x21 : iova_align(): 0.00 : ffff800010714484: and x20, x20, x1 : __iommu_dma_map(): 0.00 : ffff800010714488: mov x1, x20 0.00 : ffff80001071448c: bl ffff800010714300 0.00 : ffff800010714490: mov x24, x0 : if (!iova) 0.00 : ffff800010714494: cbz x0, ffff8000107144c8 <__iommu_dma_map+0xb8> : return DMA_MAPPING_ERROR; : : if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) { 1.92 : ffff800010714498: sub x2, x22, x19 0.00 : ffff80001071449c: mov x1, x0 0.00 : ffff8000107144a0: mov w4, w23 0.00 : ffff8000107144a4: mov x3, x20 0.00 : ffff8000107144a8: mov x0, x21 : iommu_dma_free_iova(cookie, iova, size); : return DMA_MAPPING_ERROR; : } : return iova + iova_off; 0.00 : ffff8000107144ac: add x19, x24, x19 : if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) { 0.00 : ffff8000107144b0: bl ffff800010712a00 0.00 : ffff8000107144b4: cbz w0, ffff8000107144cc <__iommu_dma_map+0xbc> : iommu_dma_free_iova(cookie, iova, size); 0.00 : ffff8000107144b8: mov x2, x20 0.00 : ffff8000107144bc: mov x1, x24 0.00 : ffff8000107144c0: mov x0, x26 0.00 : ffff8000107144c4: bl ffff800010713a40 : return DMA_MAPPING_ERROR; 0.00 : ffff8000107144c8: mov x19, #0xffffffffffffffff // #-1 : } 1.94 : ffff8000107144cc: mov x0, x19 1.94 : ffff8000107144d0: ldp x19, x20, [sp, #16] 7.70 : ffff8000107144d4: ldp x21, x22, [sp, #32] 3.76 : ffff8000107144d8: ldp x23, x24, [sp, #48] 0.00 : ffff8000107144dc: ldp x25, x26, [sp, #64] 3.83 : ffff8000107144e0: ldp x29, x30, [sp], #80 0.00 : ffff8000107144e4: ret Percent | Source code & Disassembly of vmlinux for cycles (62 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010489938 : : iov_iter_npages(): : #endif : } : EXPORT_SYMBOL(hash_and_copy_to_iter); : : int iov_iter_npages(const struct iov_iter *i, int maxpages) : { 11.29 : ffff800010489938: stp x29, x30, [sp, #-48]! 0.00 : ffff80001048993c: mov x29, sp 6.46 : ffff800010489940: str x19, [sp, #16] : size_t size = i->count; 9.89 : ffff800010489944: ldr x7, [x0, #16] : int npages = 0; : : if (!size) 0.00 : ffff800010489948: cbz x7, ffff8000104899e8 0.00 : ffff80001048994c: str x20, [x29, #24] 0.00 : ffff800010489950: mov x20, x0 : iov_iter_is_discard(): : return iov_iter_type(i) == ITER_PIPE; : } : : static inline bool iov_iter_is_discard(const struct iov_iter *i) : { : return iov_iter_type(i) == ITER_DISCARD; 6.44 : ffff800010489954: ldr w3, [x0] : iov_iter_type(): : return i->type & ~(READ | WRITE); 0.00 : ffff800010489958: and w4, w3, #0xfffffffe : iov_iter_npages(): : return 0; : if (unlikely(iov_iter_is_discard(i))) 0.00 : ffff80001048995c: cmp w4, #0x40 0.00 : ffff800010489960: b.eq ffff800010489afc // b.none 0.00 : ffff800010489964: str x21, [x29, #32] : return 0; : : if (unlikely(iov_iter_is_pipe(i))) { 0.00 : ffff800010489968: cmp w4, #0x20 0.00 : ffff80001048996c: mov w21, w1 0.00 : ffff800010489970: b.eq ffff800010489b04 // b.none : data_start(i, &iter_head, &off); : /* some of this one + all after this one */ : npages = pipe_space_for_user(iter_head, pipe->tail, pipe); : if (npages >= maxpages) : return maxpages; : } else iterate_all_kinds(i, size, v, ({ 4.81 : ffff800010489974: ldr x2, [x0, #8] 0.00 : ffff800010489978: mov w4, #0x0 // #0 : int npages = 0; 0.00 : ffff80001048997c: mov w19, #0x0 // #0 : } else iterate_all_kinds(i, size, v, ({ 0.00 : ffff800010489980: mov w8, #0x1000 // #4096 1.63 : ffff800010489984: tbnz w3, #4, ffff800010489a2c 0.00 : ffff800010489988: tbnz w3, #3, ffff800010489b94 0.00 : ffff80001048998c: tbnz w3, #6, ffff8000104899e4 0.00 : ffff800010489990: ldr x1, [x0, #24] 3.24 : ffff800010489994: ldr x0, [x1, #8] 0.00 : ffff800010489998: sub x0, x0, x2 0.00 : ffff80001048999c: cmp x0, x7 0.00 : ffff8000104899a0: csel x0, x0, x7, ls // ls = plast 6.40 : ffff8000104899a4: cbz x0, ffff800010489aa4 0.00 : ffff8000104899a8: ldr x19, [x1] 0.00 : ffff8000104899ac: add x2, x19, x2 0.00 : ffff8000104899b0: add x19, x0, x2 0.00 : ffff8000104899b4: add x19, x19, #0xfff 21.15 : ffff8000104899b8: lsr x2, x2, #12 0.00 : ffff8000104899bc: lsr x19, x19, #12 0.00 : ffff8000104899c0: sub w19, w19, w2 0.00 : ffff8000104899c4: cmp w19, w21 0.00 : ffff8000104899c8: b.lt ffff800010489ae0 // b.tstop 0.00 : ffff8000104899cc: mov w19, w21 : if (npages >= maxpages) : return maxpages; : }) : ) : return npages; : } 0.00 : ffff8000104899d0: mov w0, w19 0.00 : ffff8000104899d4: ldr x19, [sp, #16] 0.00 : ffff8000104899d8: ldp x20, x21, [x29, #24] 0.00 : ffff8000104899dc: ldp x29, x30, [sp], #48 0.00 : ffff8000104899e0: ret 0.00 : ffff8000104899e4: ldp x20, x21, [x29, #24] : return 0; 0.00 : ffff8000104899e8: mov w19, #0x0 // #0 : } 0.00 : ffff8000104899ec: mov w0, w19 1.64 : ffff8000104899f0: ldr x19, [sp, #16] 1.63 : ffff8000104899f4: ldp x29, x30, [sp], #48 0.00 : ffff8000104899f8: ret : bvec_iter_advance(): : "Attempted to advance past end of bvec iter\n")) { : iter->bi_size = 0; : return false; : } : : iter->bi_size -= bytes; 0.00 : ffff8000104899fc: sub w7, w7, w5 : bytes += iter->bi_bvec_done; : : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010489a00: adds w2, w2, w5 0.00 : ffff800010489a04: b.ne ffff800010489a24 // b.any 0.00 : ffff800010489a08: b ffff800010489a2c 0.00 : ffff800010489a0c: subs w2, w2, w3 : bytes -= bv[idx].bv_len; : idx++; 0.00 : ffff800010489a10: add w4, w4, #0x1 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010489a14: b.eq ffff800010489a2c // b.none 0.00 : ffff800010489a18: ubfiz x3, x4, #4, #32 0.00 : ffff800010489a1c: add x3, x0, x3 0.00 : ffff800010489a20: ldr w3, [x3, #8] 0.00 : ffff800010489a24: cmp w3, w2 0.00 : ffff800010489a28: b.ls ffff800010489a0c // b.plast : iov_iter_npages(): : } else iterate_all_kinds(i, size, v, ({ 0.00 : ffff800010489a2c: cbz w7, ffff800010489ae8 0.00 : ffff800010489a30: ldr x0, [x20, #24] 0.00 : ffff800010489a34: ubfiz x1, x4, #4, #32 0.00 : ffff800010489a38: add x1, x0, x1 0.00 : ffff800010489a3c: ldp w3, w5, [x1, #8] 0.00 : ffff800010489a40: sub w6, w3, w2 0.00 : ffff800010489a44: add w5, w2, w5 0.00 : ffff800010489a48: cmp w6, w7 0.00 : ffff800010489a4c: and w5, w5, #0xfff 0.00 : ffff800010489a50: sub w5, w8, w5 0.00 : ffff800010489a54: csel w6, w6, w7, ls // ls = plast 0.00 : ffff800010489a58: cmp w5, w6 0.00 : ffff800010489a5c: csel w5, w5, w6, ls // ls = plast 0.00 : ffff800010489a60: cbz w5, ffff8000104899fc 0.00 : ffff800010489a64: add w19, w19, #0x1 0.00 : ffff800010489a68: cmp w21, w19 0.00 : ffff800010489a6c: b.le ffff8000104899cc : bvec_iter_advance(): : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff800010489a70: cmp w7, w5 0.00 : ffff800010489a74: b.cs ffff8000104899fc // b.hs, b.nlast 0.00 : ffff800010489a78: adrp x1, ffff800011ad8000 0.00 : ffff800010489a7c: ldrb w0, [x1, #1302] 0.00 : ffff800010489a80: cbnz w0, ffff800010489ae8 0.00 : ffff800010489a84: mov w2, #0x1 // #1 0.00 : ffff800010489a88: adrp x0, ffff8000111e9000 0.00 : ffff800010489a8c: strb w2, [x1, #1302] 0.00 : ffff800010489a90: add x0, x0, #0x680 0.00 : ffff800010489a94: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010489a98: brk #0x800 0.00 : ffff800010489a9c: ldp x20, x21, [x29, #24] 0.00 : ffff800010489aa0: b ffff8000104899ec : iov_iter_npages(): : int npages = 0; 0.00 : ffff800010489aa4: mov w19, #0x0 // #0 : } else iterate_all_kinds(i, size, v, ({ 0.00 : ffff800010489aa8: add x1, x1, #0x10 0.00 : ffff800010489aac: ldr x0, [x1, #8] 0.00 : ffff800010489ab0: cmp x0, x7 0.00 : ffff800010489ab4: csel x0, x0, x7, ls // ls = plast 0.00 : ffff800010489ab8: cbz x0, ffff800010489aa8 0.00 : ffff800010489abc: ldr x2, [x1] 0.00 : ffff800010489ac0: add x3, x2, #0xfff 0.00 : ffff800010489ac4: add x3, x3, x0 0.00 : ffff800010489ac8: lsr x2, x2, #12 0.00 : ffff800010489acc: sub w19, w19, w2 0.00 : ffff800010489ad0: lsr x2, x3, #12 0.00 : ffff800010489ad4: add w19, w19, w2 0.00 : ffff800010489ad8: cmp w21, w19 0.00 : ffff800010489adc: b.le ffff8000104899cc 1.63 : ffff800010489ae0: subs x7, x7, x0 0.00 : ffff800010489ae4: b.ne ffff800010489aa8 // b.any : } 14.12 : ffff800010489ae8: mov w0, w19 0.00 : ffff800010489aec: ldr x19, [sp, #16] 1.64 : ffff800010489af0: ldp x20, x21, [x29, #24] 8.03 : ffff800010489af4: ldp x29, x30, [sp], #48 0.00 : ffff800010489af8: ret 0.00 : ffff800010489afc: ldr x20, [x29, #24] 0.00 : ffff800010489b00: b ffff8000104899e8 0.00 : ffff800010489b04: str x22, [x29, #40] : struct pipe_inode_info *pipe = i->pipe; 0.00 : ffff800010489b08: ldr x22, [x0, #24] : if (!sanity(i)) 0.00 : ffff800010489b0c: bl ffff800010487f20 0.00 : ffff800010489b10: tst w0, #0xff 0.00 : ffff800010489b14: b.eq ffff800010489c14 // b.none : data_start(): : size_t off = i->iov_offset; 0.00 : ffff800010489b18: ldr x2, [x20, #8] : unsigned int iter_head = i->head; 0.00 : ffff800010489b1c: ldr w0, [x20, #32] : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || 0.00 : ffff800010489b20: cbz x2, ffff800010489b5c : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff800010489b24: ldr x4, [x20, #24] : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || 0.00 : ffff800010489b28: mov w6, #0x28 // #40 0.00 : ffff800010489b2c: adrp x3, ffff800010d18000 0.00 : ffff800010489b30: add x3, x3, #0xf80 0.00 : ffff800010489b34: mov x5, #0x1000 // #4096 : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff800010489b38: ldr w1, [x4, #92] : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || 0.00 : ffff800010489b3c: ldr x4, [x4, #144] : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff800010489b40: sub w1, w1, #0x1 : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || 0.00 : ffff800010489b44: and w1, w1, w0 0.00 : ffff800010489b48: umaddl x1, w1, w6, x4 0.00 : ffff800010489b4c: ldr x1, [x1, #16] 0.00 : ffff800010489b50: cmp x1, x3 : iter_head++; 0.00 : ffff800010489b54: ccmp x2, x5, #0x4, eq // eq = none 0.00 : ffff800010489b58: cinc w0, w0, eq // eq = none : iov_iter_npages(): : npages = pipe_space_for_user(iter_head, pipe->tail, pipe); 0.00 : ffff800010489b5c: ldp w2, w1, [x22, #84] : pipe_space_for_user(): : struct pipe_inode_info *pipe) : { : unsigned int p_occupancy, p_space; : : p_occupancy = pipe_occupancy(head, tail); : if (p_occupancy >= pipe->max_usage) 0.00 : ffff800010489b60: mov w19, #0x0 // #0 : pipe_occupancy(): : return head - tail; 0.00 : ffff800010489b64: sub w0, w0, w2 : pipe_space_for_user(): : if (p_occupancy >= pipe->max_usage) 0.00 : ffff800010489b68: cmp w1, w0 0.00 : ffff800010489b6c: b.ls ffff800010489b80 // b.plast : return 0; : p_space = pipe->ring_size - p_occupancy; 0.00 : ffff800010489b70: ldr w19, [x22, #92] 0.00 : ffff800010489b74: sub w0, w19, w0 0.00 : ffff800010489b78: cmp w0, w1 0.00 : ffff800010489b7c: csel w19, w0, w1, ls // ls = plast 0.00 : ffff800010489b80: cmp w21, w19 : iov_iter_npages(): 0.00 : ffff800010489b84: ldr x22, [x29, #40] 0.00 : ffff800010489b88: csel w19, w21, w19, le 0.00 : ffff800010489b8c: ldp x20, x21, [x29, #24] 0.00 : ffff800010489b90: b ffff8000104899ec : } else iterate_all_kinds(i, size, v, ({ 0.00 : ffff800010489b94: ldr x3, [x0, #24] 0.00 : ffff800010489b98: ldr x0, [x3, #8] 0.00 : ffff800010489b9c: sub x0, x0, x2 0.00 : ffff800010489ba0: cmp x0, x7 0.00 : ffff800010489ba4: csel x0, x0, x7, ls // ls = plast 0.00 : ffff800010489ba8: cbz x0, ffff800010489c20 0.00 : ffff800010489bac: ldr x1, [x3] 0.00 : ffff800010489bb0: add x2, x1, x2 0.00 : ffff800010489bb4: add x19, x0, x2 0.00 : ffff800010489bb8: add x19, x19, #0xfff 0.00 : ffff800010489bbc: lsr x2, x2, #12 0.00 : ffff800010489bc0: lsr x19, x19, #12 0.00 : ffff800010489bc4: sub w19, w19, w2 0.00 : ffff800010489bc8: cmp w21, w19 0.00 : ffff800010489bcc: b.le ffff8000104899cc 0.00 : ffff800010489bd0: subs x7, x7, x0 0.00 : ffff800010489bd4: b.eq ffff800010489ae8 // b.none 0.00 : ffff800010489bd8: add x3, x3, #0x10 0.00 : ffff800010489bdc: ldr x0, [x3, #8] 0.00 : ffff800010489be0: cmp x0, x7 0.00 : ffff800010489be4: csel x0, x0, x7, ls // ls = plast 0.00 : ffff800010489be8: cbz x0, ffff800010489bd8 0.00 : ffff800010489bec: ldr x1, [x3] 0.00 : ffff800010489bf0: add x2, x1, #0xfff 0.00 : ffff800010489bf4: add x2, x2, x0 0.00 : ffff800010489bf8: lsr x1, x1, #12 0.00 : ffff800010489bfc: sub w19, w19, w1 0.00 : ffff800010489c00: lsr x1, x2, #12 0.00 : ffff800010489c04: add w19, w19, w1 0.00 : ffff800010489c08: cmp w21, w19 0.00 : ffff800010489c0c: b.gt ffff800010489bd0 0.00 : ffff800010489c10: b ffff8000104899cc 0.00 : ffff800010489c14: ldp x20, x21, [x29, #24] 0.00 : ffff800010489c18: ldr x22, [x29, #40] 0.00 : ffff800010489c1c: b ffff8000104899e8 : int npages = 0; 0.00 : ffff800010489c20: mov w19, #0x0 // #0 0.00 : ffff800010489c24: b ffff800010489bd8 Percent | Source code & Disassembly of vmlinux for cycles (39 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001071d260 : : arm_smmu_map(): : } : : static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, : phys_addr_t paddr, size_t size, int prot, gfp_t gfp) : { : struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 7.87 : ffff80001071d260: ldur x0, [x0, #-96] : : if (!ops) 0.00 : ffff80001071d264: cbz x0, ffff80001071d280 : { 7.83 : ffff80001071d268: stp x29, x30, [sp, #-16]! 0.00 : ffff80001071d26c: mov x29, sp : return -ENODEV; : : return ops->map(ops, iova, paddr, size, prot); 4.98 : ffff80001071d270: ldr x5, [x0] 0.00 : ffff80001071d274: blr x5 : } 76.72 : ffff80001071d278: ldp x29, x30, [sp], #16 2.60 : ffff80001071d27c: ret : return -ENODEV; 0.00 : ffff80001071d280: mov w0, #0xffffffed // #-19 : } 0.00 : ffff80001071d284: ret Percent | Source code & Disassembly of vmlinux for cycles (20 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011b4b0 : : account_system_index_time(): : * @cputime: the CPU time spent in kernel space since the last update : * @index: pointer to cpustat field that has to be updated : */ : void account_system_index_time(struct task_struct *p, : u64 cputime, enum cpu_usage_stat index) : { 0.00 : ffff80001011b4b0: stp x29, x30, [sp, #-48]! 0.00 : ffff80001011b4b4: mov x29, sp 0.00 : ffff80001011b4b8: stp x19, x20, [sp, #16] 0.00 : ffff80001011b4bc: mov x19, x0 5.07 : ffff80001011b4c0: str x21, [sp, #32] 0.00 : ffff80001011b4c4: mov x20, x1 0.00 : ffff80001011b4c8: mov w21, w2 : /* Add system time to process. */ : p->stime += cputime; 0.00 : ffff80001011b4cc: ldr x1, [x0, #1368] : get_running_cputimer(): : */ : #ifdef CONFIG_POSIX_TIMERS : static inline : struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk) : { : struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 10.00 : ffff80001011b4d0: ldr x0, [x0, #1640] : account_system_index_time(): 0.00 : ffff80001011b4d4: add x1, x1, x20 0.00 : ffff80001011b4d8: str x1, [x19, #1368] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011b4dc: ldr w1, [x0, #344] : get_running_cputimer(): : : /* : * Check whether posix CPU timers are active. If not the thread : * group accounting is not active either. Lockless check. : */ : if (!READ_ONCE(tsk->signal->posix_cputimers.timers_active)) 0.00 : ffff80001011b4e0: cbz w1, ffff80001011b510 : account_group_system_time(): : static inline void account_group_system_time(struct task_struct *tsk, : u64 cputime) : { : struct thread_group_cputimer *cputimer = get_running_cputimer(tsk); : : if (!cputimer) 0.00 : ffff80001011b4e4: ldr x2, [x19, #1648] : get_running_cputimer(): : struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; 0.00 : ffff80001011b4e8: add x1, x0, #0xf8 : account_group_system_time(): : if (!cputimer) 0.00 : ffff80001011b4ec: cmp x2, #0x0 0.00 : ffff80001011b4f0: ccmp x1, #0x0, #0x4, ne // ne = any 0.00 : ffff80001011b4f4: b.eq ffff80001011b510 // b.none : return; : : atomic64_add(cputime, &cputimer->cputime_atomic.stime); 0.00 : ffff80001011b4f8: add x1, x0, #0x100 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011b4fc: b ffff80001011b578 0.00 : ffff80001011b500: b ffff80001011b578 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff80001011b504: mov x2, x20 0.00 : ffff80001011b508: stadd x2, [x1] 0.00 : ffff80001011b50c: nop : task_group_account_field(): : __this_cpu_add(kernel_cpustat.cpustat[index], tmp); 20.12 : ffff80001011b510: adrp x3, ffff80001151e000 0.00 : ffff80001011b514: add x3, x3, #0xfc0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011b518: mrs x5, tpidr_el1 : task_group_account_field(): 0.00 : ffff80001011b51c: add x3, x3, w21, sxtw #3 : cgroup_account_cputime_field(): : enum cpu_usage_stat index, : u64 delta_exec) : { : struct cgroup *cgrp; : : cpuacct_account_field(task, index, delta_exec); 0.00 : ffff80001011b520: mov w1, w21 : task_group_account_field(): 0.00 : ffff80001011b524: ldr x4, [x3, x5] : cgroup_account_cputime_field(): 0.00 : ffff80001011b528: mov x0, x19 0.00 : ffff80001011b52c: mov x2, x20 : task_group_account_field(): 0.00 : ffff80001011b530: add x4, x4, x20 0.00 : ffff80001011b534: str x4, [x3, x5] : cgroup_account_cputime_field(): 0.00 : ffff80001011b538: bl ffff800010136508 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff80001011b53c: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): 0.00 : ffff80001011b540: ldr x0, [x19, #2000] : task_dfl_cgroup(): : return task_css_set(task)->dfl_cgrp; 4.83 : ffff80001011b544: ldr x0, [x0, #88] : cgroup_parent(): : if (parent_css) 14.94 : ffff80001011b548: ldr x1, [x0, #232] 0.00 : ffff80001011b54c: cbz x1, ffff80001011b55c : cgroup_account_cputime_field(): : : rcu_read_lock(); : cgrp = task_dfl_cgroup(task); : if (cgroup_parent(cgrp)) : __cgroup_account_cputime_field(cgrp, index, delta_exec); 0.00 : ffff80001011b550: mov x2, x20 0.00 : ffff80001011b554: mov w1, w21 0.00 : ffff80001011b558: bl ffff800010197428 <__cgroup_account_cputime_field> : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 35.03 : ffff80001011b55c: bl ffff800010160d20 <__rcu_read_unlock> : account_system_index_time(): : : /* Add system time to cpustat. */ : task_group_account_field(p, index, cputime); : : /* Account for system time used */ : acct_account_cputime(p); 0.00 : ffff80001011b560: mov x0, x19 0.00 : ffff80001011b564: bl ffff8000101b4258 : } 4.98 : ffff80001011b568: ldp x19, x20, [sp, #16] 5.02 : ffff80001011b56c: ldr x21, [sp, #32] 0.00 : ffff80001011b570: ldp x29, x30, [sp], #48 0.00 : ffff80001011b574: ret : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff80001011b578: add x0, x0, #0x100 0.00 : ffff80001011b57c: b ffff80001011bd9c 0.00 : ffff80001011b580: b ffff80001011b510 Percent | Source code & Disassembly of vmlinux for cycles (22 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011b668 : : irqtime_account_process_tick(): : * p->stime and friends are only updated on system time and not on irq : * softirq as those do not count in task exec_runtime any more. : */ : static void irqtime_account_process_tick(struct task_struct *p, int user_tick, : int ticks) : { 0.00 : ffff80001011b668: stp x29, x30, [sp, #-48]! 0.00 : ffff80001011b66c: mov x29, sp 0.00 : ffff80001011b670: stp x19, x20, [sp, #16] 0.00 : ffff80001011b674: mov x19, x0 4.83 : ffff80001011b678: str x21, [sp, #32] 0.00 : ffff80001011b67c: mov w20, w1 0.00 : ffff80001011b680: mov w21, w2 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011b684: nop 0.00 : ffff80001011b688: mov x2, sp 0.00 : ffff80001011b68c: mov x0, #0xffffffffffffffff // #-1 : steal_account_process_time(): : return 0; 0.00 : ffff80001011b690: mov x4, #0x0 // #0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 4.79 : ffff80001011b694: mrs x3, tpidr_el1 : irqtime_tick_accounted(): : struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); 0.00 : ffff80001011b698: adrp x1, ffff80001151f000 : irqtime_account_process_tick(): : u64 other, cputime = TICK_NSEC * ticks; 0.00 : ffff80001011b69c: mov w2, #0x900 // #2304 : irqtime_tick_accounted(): : struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); 0.00 : ffff80001011b6a0: add x1, x1, #0x40 : irqtime_account_process_tick(): : u64 other, cputime = TICK_NSEC * ticks; 0.00 : ffff80001011b6a4: movk w2, #0x3d, lsl #16 : irqtime_tick_accounted(): : struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); 0.00 : ffff80001011b6a8: add x1, x1, x3 : irqtime_account_process_tick(): : u64 other, cputime = TICK_NSEC * ticks; 0.00 : ffff80001011b6ac: smull x21, w21, w2 : irqtime_tick_accounted(): : delta = min(irqtime->tick_delta, maxtime); 0.00 : ffff80001011b6b0: ldr x2, [x1, #8] 0.00 : ffff80001011b6b4: cmp x2, x0 0.00 : ffff80001011b6b8: csel x0, x2, x0, ls // ls = plast : irqtime->tick_delta -= delta; 0.00 : ffff80001011b6bc: sub x2, x2, x0 4.76 : ffff80001011b6c0: str x2, [x1, #8] : account_other_time(): : accounted += irqtime_tick_accounted(max - accounted); 0.00 : ffff80001011b6c4: add x4, x4, x0 : irqtime_account_process_tick(): : * Subtract those ticks from the amount of time accounted to : * idle, or potentially user or system time. Due to rounding, : * other time can exceed ticks occasionally. : */ : other = account_other_time(ULONG_MAX); : if (other >= cputime) 0.00 : ffff80001011b6c8: cmp x21, x4 0.00 : ffff80001011b6cc: b.ls ffff80001011b750 // b.plast 0.00 : ffff80001011b6d0: str x22, [x29, #40] : return; : : cputime -= other; 0.00 : ffff80001011b6d4: sub x21, x21, x4 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 13.97 : ffff80001011b6d8: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011b6dc: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001011b6e0: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001011b6e4: str w0, [x1, #16] : this_cpu_ksoftirqd(): : : DECLARE_PER_CPU(struct task_struct *, ksoftirqd); : : static inline struct task_struct *this_cpu_ksoftirqd(void) : { : return this_cpu_read(ksoftirqd); 0.00 : ffff80001011b6e8: adrp x0, ffff80001151e000 0.00 : ffff80001011b6ec: add x0, x0, #0xf80 : __my_cpu_offset(): 0.00 : ffff80001011b6f0: mrs x2, tpidr_el1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011b6f4: ldr x22, [x0, x2] 9.53 : ffff80001011b6f8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001011b6fc: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001011b700: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001011b704: cbnz x0, ffff80001011b7c4 : this_cpu_ksoftirqd(): 0.00 : ffff80001011b708: bl ffff800010cd2e78 : irqtime_account_process_tick(): : : if (this_cpu_ksoftirqd() == p) { 4.94 : ffff80001011b70c: cmp x19, x22 0.00 : ffff80001011b710: b.eq ffff80001011b7f8 // b.none : * ksoftirqd time do not get accounted in cpu_softirq_time. : * So, we have to handle it separately here. : * Also, p->stime needs to be updated for ksoftirqd. : */ : account_system_index_time(p, cputime, CPUTIME_SOFTIRQ); : } else if (user_tick) { 0.00 : ffff80001011b714: cbnz w20, ffff80001011b7d0 : account_user_time(p, cputime); : } else if (p == this_rq()->idle) { 0.00 : ffff80001011b718: adrp x0, ffff800011528000 0.00 : ffff80001011b71c: add x0, x0, #0xe80 : __my_cpu_offset(): 0.00 : ffff80001011b720: mrs x1, tpidr_el1 : irqtime_account_process_tick(): 0.00 : ffff80001011b724: add x0, x0, x1 0.00 : ffff80001011b728: ldr x0, [x0, #2360] 0.00 : ffff80001011b72c: cmp x0, x19 0.00 : ffff80001011b730: b.eq ffff80001011b810 // b.none : account_idle_time(cputime); : } else if (p->flags & PF_VCPU) { /* System time or guest time */ 4.53 : ffff80001011b734: ldr w0, [x19, #44] 0.00 : ffff80001011b738: tbnz w0, #4, ffff80001011b7e4 : account_guest_time(p, cputime); : } else { : account_system_index_time(p, cputime, CPUTIME_SYSTEM); 0.00 : ffff80001011b73c: mov w2, #0x2 // #2 0.00 : ffff80001011b740: mov x1, x21 0.00 : ffff80001011b744: mov x0, x19 0.00 : ffff80001011b748: bl ffff80001011b4b0 14.53 : ffff80001011b74c: ldr x22, [x29, #40] : } : } 9.39 : ffff80001011b750: ldp x19, x20, [sp, #16] 4.71 : ffff80001011b754: ldr x21, [sp, #32] 0.00 : ffff80001011b758: ldp x29, x30, [sp], #48 0.00 : ffff80001011b75c: ret : paravirt_steal_clock(): : : extern struct paravirt_patch_template pv_ops; : : static inline u64 paravirt_steal_clock(int cpu) : { : return pv_ops.time.steal_clock(cpu); 0.00 : ffff80001011b760: adrp x1, ffff800011af8000 : steal_account_process_time(): : steal = paravirt_steal_clock(smp_processor_id()); 0.00 : ffff80001011b764: adrp x0, ffff80001151d000 : __my_cpu_offset(): 0.00 : ffff80001011b768: mrs x2, tpidr_el1 : paravirt_steal_clock(): 0.00 : ffff80001011b76c: ldr x1, [x1, #248] : steal_account_process_time(): 0.00 : ffff80001011b770: add x0, x0, #0x18 : paravirt_steal_clock(): 0.00 : ffff80001011b774: ldr w0, [x0, x2] 0.00 : ffff80001011b778: blr x1 : steal_account_process_time(): : steal -= this_rq()->prev_steal_time; 0.00 : ffff80001011b77c: adrp x3, ffff800011528000 0.00 : ffff80001011b780: add x3, x3, #0xe80 : __my_cpu_offset(): 0.00 : ffff80001011b784: mrs x1, tpidr_el1 : steal_account_process_time(): 0.00 : ffff80001011b788: add x1, x3, x1 0.00 : ffff80001011b78c: ldr x4, [x1, #2848] 0.00 : ffff80001011b790: sub x4, x0, x4 : account_steal_time(steal); 0.00 : ffff80001011b794: mov x0, x4 0.00 : ffff80001011b798: bl ffff80001011b600 : __my_cpu_offset(): : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff80001011b79c: mov x2, sp : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011b7a0: mrs x0, tpidr_el1 : steal_account_process_time(): : this_rq()->prev_steal_time += steal; 0.00 : ffff80001011b7a4: add x3, x3, x0 : account_other_time(): : if (accounted < max) 0.00 : ffff80001011b7a8: cmn x4, #0x1 : steal_account_process_time(): : this_rq()->prev_steal_time += steal; 0.00 : ffff80001011b7ac: ldr x0, [x3, #2848] 0.00 : ffff80001011b7b0: add x0, x0, x4 0.00 : ffff80001011b7b4: str x0, [x3, #2848] : account_other_time(): : if (accounted < max) 0.00 : ffff80001011b7b8: b.eq ffff80001011b750 // b.none 0.00 : ffff80001011b7bc: mvn x0, x4 0.00 : ffff80001011b7c0: b ffff80001011b694 : __read_once_size(): : __READ_ONCE_SIZE; 4.65 : ffff80001011b7c4: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001011b7c8: cbz x0, ffff80001011b708 19.36 : ffff80001011b7cc: b ffff80001011b70c : irqtime_account_process_tick(): : account_user_time(p, cputime); 0.00 : ffff80001011b7d0: mov x1, x21 0.00 : ffff80001011b7d4: mov x0, x19 0.00 : ffff80001011b7d8: bl ffff80001011b308 0.00 : ffff80001011b7dc: ldr x22, [x29, #40] 0.00 : ffff80001011b7e0: b ffff80001011b750 : account_guest_time(p, cputime); 0.00 : ffff80001011b7e4: mov x1, x21 0.00 : ffff80001011b7e8: mov x0, x19 0.00 : ffff80001011b7ec: bl ffff80001011b3e8 0.00 : ffff80001011b7f0: ldr x22, [x29, #40] 0.00 : ffff80001011b7f4: b ffff80001011b750 : account_system_index_time(p, cputime, CPUTIME_SOFTIRQ); 0.00 : ffff80001011b7f8: mov w2, #0x3 // #3 0.00 : ffff80001011b7fc: mov x1, x21 0.00 : ffff80001011b800: mov x0, x19 0.00 : ffff80001011b804: bl ffff80001011b4b0 0.00 : ffff80001011b808: ldr x22, [x29, #40] 0.00 : ffff80001011b80c: b ffff80001011b750 : account_idle_time(cputime); 0.00 : ffff80001011b810: mov x0, x21 0.00 : ffff80001011b814: bl ffff80001011b620 0.00 : ffff80001011b818: ldr x22, [x29, #40] 0.00 : ffff80001011b81c: b ffff80001011b750 Percent | Source code & Disassembly of vmlinux for cycles (68 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001045c978 : : blk_queue_is_zoned(): : return q->limits.zoned; : } : : static inline bool blk_queue_is_zoned(struct request_queue *q) : { : switch (blk_queue_zoned_model(q)) { 7.36 : ffff80001045c978: ldr w4, [x0, #1140] 0.00 : ffff80001045c97c: sub w4, w4, #0x1 7.34 : ffff80001045c980: cmp w4, #0x1 0.00 : ffff80001045c984: b.hi ffff80001045c990 // b.pmore : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 0.00 : ffff80001045c988: ldr w4, [x1, #16] : blk_mq_plug(): : { : /* : * For regular block devices or read operations, use the context plug : * which may be NULL if blk_start_plug() was not executed. : */ : if (!blk_queue_is_zoned(q) || !op_is_write(bio_op(bio))) 0.00 : ffff80001045c98c: tbnz w4, #0, ffff80001045ca94 : blk_attempt_plug_merge(): : * : * Caller must ensure !blk_queue_nomerges(q) beforehand. : */ : bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, : unsigned int nr_segs, struct request **same_queue_rq) : { 0.00 : ffff80001045c990: stp x29, x30, [sp, #-80]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 27.80 : ffff80001045c994: mrs x4, sp_el0 : blk_attempt_plug_merge(): 0.00 : ffff80001045c998: mov x29, sp 0.00 : ffff80001045c99c: str x21, [sp, #32] : blk_mq_plug(): : return current->plug; 1.48 : ffff80001045c9a0: ldr x21, [x4, #1840] : blk_attempt_plug_merge(): : struct blk_plug *plug; : struct request *rq; : struct list_head *plug_list; : : plug = blk_mq_plug(q, bio); : if (!plug) 0.00 : ffff80001045c9a4: cbz x21, ffff80001045caa0 11.79 : ffff80001045c9a8: str x19, [x29, #16] : return false; : : plug_list = &plug->mq_list; : : list_for_each_entry_reverse(rq, plug_list, queuelist) { 2.87 : ffff80001045c9ac: ldr x4, [x21, #8] 0.00 : ffff80001045c9b0: sub x19, x4, #0x48 0.00 : ffff80001045c9b4: cmp x21, x4 0.00 : ffff80001045c9b8: b.eq ffff80001045ca9c // b.none 0.00 : ffff80001045c9bc: cmp x3, #0x0 0.00 : ffff80001045c9c0: str x20, [x29, #24] 0.00 : ffff80001045c9c4: stp x22, x23, [x29, #40] 0.00 : ffff80001045c9c8: mov x20, x0 0.00 : ffff80001045c9cc: stp x24, x25, [x29, #56] 0.00 : ffff80001045c9d0: mov x23, x1 0.00 : ffff80001045c9d4: mov x24, x3 0.00 : ffff80001045c9d8: mov w25, w2 0.00 : ffff80001045c9dc: cset w22, ne // ne = any 0.00 : ffff80001045c9e0: b ffff80001045c9f4 0.00 : ffff80001045c9e4: ldr x4, [x19, #80] 0.00 : ffff80001045c9e8: sub x19, x4, #0x48 0.00 : ffff80001045c9ec: cmp x21, x4 0.00 : ffff80001045c9f0: b.eq ffff80001045cab0 // b.none : bool merged = false; : : if (rq->q == q && same_queue_rq) { 0.00 : ffff80001045c9f4: ldr x4, [x19] 0.00 : ffff80001045c9f8: cmp w22, #0x0 0.00 : ffff80001045c9fc: ccmp x4, x20, #0x0, ne // ne = any 0.00 : ffff80001045ca00: b.eq ffff80001045ca6c // b.none : * rq in a queue : **/ : *same_queue_rq = rq; : } : : if (rq->q != q || !blk_rq_merge_ok(rq, bio)) 0.00 : ffff80001045ca04: cmp x20, x4 0.00 : ffff80001045ca08: b.ne ffff80001045c9e4 // b.any 0.00 : ffff80001045ca0c: mov x1, x23 0.00 : ffff80001045ca10: mov x0, x19 0.00 : ffff80001045ca14: bl ffff800010462d60 0.00 : ffff80001045ca18: tst w0, #0xff 0.00 : ffff80001045ca1c: b.eq ffff80001045c9e4 // b.none : continue; : : switch (blk_try_merge(rq, bio)) { 0.00 : ffff80001045ca20: mov x1, x23 0.00 : ffff80001045ca24: mov x0, x19 0.00 : ffff80001045ca28: bl ffff800010462eb8 0.00 : ffff80001045ca2c: cmp w0, #0x2 0.00 : ffff80001045ca30: b.eq ffff80001045cac0 // b.none 0.00 : ffff80001045ca34: cmp w0, #0x3 0.00 : ffff80001045ca38: b.eq ffff80001045ca78 // b.none 0.00 : ffff80001045ca3c: cmp w0, #0x1 0.00 : ffff80001045ca40: b.ne ffff80001045c9e4 // b.any : case ELEVATOR_BACK_MERGE: : merged = bio_attempt_back_merge(rq, bio, nr_segs); : break; : case ELEVATOR_FRONT_MERGE: : merged = bio_attempt_front_merge(rq, bio, nr_segs); 0.00 : ffff80001045ca44: mov w2, w25 0.00 : ffff80001045ca48: mov x1, x23 0.00 : ffff80001045ca4c: mov x0, x19 0.00 : ffff80001045ca50: bl ffff80001045c710 0.00 : ffff80001045ca54: and w0, w0, #0xff : break; : default: : break; : } : : if (merged) 0.00 : ffff80001045ca58: cbz w0, ffff80001045c9e4 0.00 : ffff80001045ca5c: ldp x19, x20, [x29, #16] 0.00 : ffff80001045ca60: ldp x22, x23, [x29, #40] 0.00 : ffff80001045ca64: ldp x24, x25, [x29, #56] 0.00 : ffff80001045ca68: b ffff80001045caa4 : *same_queue_rq = rq; 0.00 : ffff80001045ca6c: str x19, [x24] 0.00 : ffff80001045ca70: ldr x4, [x19] 0.00 : ffff80001045ca74: b ffff80001045ca04 : merged = bio_attempt_discard_merge(q, rq, bio); 0.00 : ffff80001045ca78: mov x2, x23 0.00 : ffff80001045ca7c: mov x1, x19 0.00 : ffff80001045ca80: mov x0, x20 0.00 : ffff80001045ca84: bl ffff80001045c7c8 0.00 : ffff80001045ca88: and w0, w0, #0xff : if (merged) 0.00 : ffff80001045ca8c: cbz w0, ffff80001045c9e4 0.00 : ffff80001045ca90: b ffff80001045ca5c : return false; 0.00 : ffff80001045ca94: mov w0, #0x0 // #0 : return true; : } : : return false; : } 0.00 : ffff80001045ca98: ret 19.15 : ffff80001045ca9c: ldr x19, [x29, #16] : return false; 19.23 : ffff80001045caa0: mov w0, #0x0 // #0 : } 0.00 : ffff80001045caa4: ldr x21, [sp, #32] 2.97 : ffff80001045caa8: ldp x29, x30, [sp], #80 0.00 : ffff80001045caac: ret 0.00 : ffff80001045cab0: ldp x19, x20, [x29, #16] 0.00 : ffff80001045cab4: ldp x22, x23, [x29, #40] 0.00 : ffff80001045cab8: ldp x24, x25, [x29, #56] 0.00 : ffff80001045cabc: b ffff80001045caa0 : merged = bio_attempt_back_merge(rq, bio, nr_segs); 0.00 : ffff80001045cac0: mov w2, w25 0.00 : ffff80001045cac4: mov x1, x23 0.00 : ffff80001045cac8: mov x0, x19 0.00 : ffff80001045cacc: bl ffff80001045c660 0.00 : ffff80001045cad0: and w0, w0, #0xff : if (merged) 0.00 : ffff80001045cad4: cbz w0, ffff80001045c9e4 0.00 : ffff80001045cad8: b ffff80001045ca5c Percent | Source code & Disassembly of vmlinux for cycles (42 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010713bb8 : : is_kdump_kernel(): : * of previous kernel. : */ : : static inline bool is_kdump_kernel(void) : { : return elfcorehdr_addr != ELFCORE_ADDR_MAX; 92.74 : ffff800010713bb8: adrp x2, ffff80001192b000 : iommu_dma_deferred_attach(): : static int iommu_dma_deferred_attach(struct device *dev, : struct iommu_domain *domain) : { : const struct iommu_ops *ops = domain->ops; : : if (!is_kdump_kernel()) 0.00 : ffff800010713bbc: ldr x2, [x2, #2184] 4.83 : ffff800010713bc0: cmn x2, #0x1 0.00 : ffff800010713bc4: b.eq ffff800010713bd4 // b.none : return 0; : : if (unlikely(ops->is_attach_deferred && 0.00 : ffff800010713bc8: ldr x2, [x1, #8] 0.00 : ffff800010713bcc: ldr x2, [x2, #176] 0.00 : ffff800010713bd0: cbnz x2, ffff800010713bdc : return 0; 2.43 : ffff800010713bd4: mov w0, #0x0 // #0 : ops->is_attach_deferred(domain, dev))) : return iommu_attach_device(domain, dev); : : return 0; : } 0.00 : ffff800010713bd8: ret : { 0.00 : ffff800010713bdc: stp x29, x30, [sp, #-32]! 0.00 : ffff800010713be0: mov x29, sp 0.00 : ffff800010713be4: stp x19, x20, [sp, #16] 0.00 : ffff800010713be8: mov x19, x1 0.00 : ffff800010713bec: mov x20, x0 : if (unlikely(ops->is_attach_deferred && 0.00 : ffff800010713bf0: mov x1, x0 0.00 : ffff800010713bf4: mov x0, x19 0.00 : ffff800010713bf8: blr x2 0.00 : ffff800010713bfc: tst w0, #0xff 0.00 : ffff800010713c00: b.ne ffff800010713c14 // b.any : return 0; 0.00 : ffff800010713c04: mov w0, #0x0 // #0 : } 0.00 : ffff800010713c08: ldp x19, x20, [sp, #16] 0.00 : ffff800010713c0c: ldp x29, x30, [sp], #32 0.00 : ffff800010713c10: ret : return iommu_attach_device(domain, dev); 0.00 : ffff800010713c14: mov x1, x20 0.00 : ffff800010713c18: mov x0, x19 0.00 : ffff800010713c1c: bl ffff800010711690 0.00 : ffff800010713c20: b ffff800010713c08 Percent | Source code & Disassembly of vmlinux for cycles (19 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010aeb488 : : arch_timer_handler_phys(): : : return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt); : } : : static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id) : { 5.53 : ffff800010aeb488: stp x29, x30, [sp, #-16]! 0.00 : ffff800010aeb48c: mov x29, sp : arch_timer_reg_read_cp15(): : u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) : { : if (access == ARCH_TIMER_PHYS_ACCESS) { : switch (reg) { : case ARCH_TIMER_REG_CTRL: : return read_sysreg(cntp_ctl_el0); 5.01 : ffff800010aeb490: mrs x2, cntp_ctl_el0 : timer_handler(): : if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { 0.00 : ffff800010aeb494: tbz w2, #2, ffff800010aeb4bc 0.00 : ffff800010aeb498: mov x0, x1 : ctrl |= ARCH_TIMER_CTRL_IT_MASK; 0.00 : ffff800010aeb49c: orr w2, w2, #0x2 : arch_timer_reg_write_cp15(): : write_sysreg(val, cntp_ctl_el0); 0.00 : ffff800010aeb4a0: msr cntp_ctl_el0, x2 : isb(); 0.00 : ffff800010aeb4a4: isb : timer_handler(): : evt->event_handler(evt); 89.46 : ffff800010aeb4a8: ldr x1, [x1] 0.00 : ffff800010aeb4ac: blr x1 : return IRQ_HANDLED; 0.00 : ffff800010aeb4b0: mov w0, #0x1 // #1 : arch_timer_handler_phys(): : struct clock_event_device *evt = dev_id; : : return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt); : } 0.00 : ffff800010aeb4b4: ldp x29, x30, [sp], #16 0.00 : ffff800010aeb4b8: ret : timer_handler(): : return IRQ_NONE; 0.00 : ffff800010aeb4bc: mov w0, #0x0 // #0 : arch_timer_handler_phys(): : } 0.00 : ffff800010aeb4c0: ldp x29, x30, [sp], #16 0.00 : ffff800010aeb4c4: ret Percent | Source code & Disassembly of vmlinux for cycles (18 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010176168 : : run_posix_cpu_timers(): : * This is called from the timer interrupt handler. The irq handler has : * already updated our counts. We need to check if any timers fire now. : * Interrupts are disabled. : */ : void run_posix_cpu_timers(void) : { 10.90 : ffff800010176168: stp x29, x30, [sp, #-160]! 0.00 : ffff80001017616c: mov x29, sp 0.00 : ffff800010176170: stp x19, x20, [sp, #16] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 5.74 : ffff800010176174: mrs x19, sp_el0 : expiry_cache_is_inactive(): : return !(~pct->bases[CPUCLOCK_PROF].nextevt | 5.51 : ffff800010176178: add x20, x19, #0x5b0 : run_posix_cpu_timers(): : { 0.00 : ffff80001017617c: str x21, [sp, #32] 5.58 : ffff800010176180: str x23, [sp, #48] 0.00 : ffff800010176184: adrp x21, ffff800011909000 0.00 : ffff800010176188: add x0, x21, #0x908 : struct task_struct *tsk = current; : struct k_itimer *timer, *next; : unsigned long flags; : LIST_HEAD(firing); 0.00 : ffff80001017618c: add x23, x29, #0x70 : expiry_cache_is_inactive(): : return !(~pct->bases[CPUCLOCK_PROF].nextevt | 0.00 : ffff800010176190: ldr x1, [x19, #1456] : run_posix_cpu_timers(): : { 11.23 : ffff800010176194: ldr x4, [x0] 0.00 : ffff800010176198: str x4, [x29, #152] 0.00 : ffff80001017619c: mov x4, #0x0 // #0 : expiry_cache_is_inactive(): : ~pct->bases[CPUCLOCK_VIRT].nextevt | 0.00 : ffff8000101761a0: ldr x2, [x20, #24] : ~pct->bases[CPUCLOCK_SCHED].nextevt); 11.23 : ffff8000101761a4: ldr x3, [x20, #48] : return !(~pct->bases[CPUCLOCK_PROF].nextevt | 0.00 : ffff8000101761a8: and x0, x1, x2 : run_posix_cpu_timers(): : LIST_HEAD(firing); 0.00 : ffff8000101761ac: stp x23, x23, [x29, #112] : expiry_cache_is_inactive(): : ~pct->bases[CPUCLOCK_VIRT].nextevt | 0.00 : ffff8000101761b0: and x0, x0, x3 : fastpath_timer_check(): : if (!expiry_cache_is_inactive(pct)) { 0.00 : ffff8000101761b4: cmn x0, #0x1 0.00 : ffff8000101761b8: b.eq ffff8000101761dc // b.none : task_cputimers_expired(): : if (samples[i] >= pct->bases[i].nextevt) 0.00 : ffff8000101761bc: ldr x5, [x19, #272] : task_cputime(): : extern u64 task_gtime(struct task_struct *t); : #else : static inline void task_cputime(struct task_struct *t, : u64 *utime, u64 *stime) : { : *utime = t->utime; 0.00 : ffff8000101761c0: ldr x4, [x19, #1360] : store_samples(): : samples[CPUCLOCK_PROF] = stime + utime; 0.00 : ffff8000101761c4: ldr x0, [x19, #1368] : task_cputimers_expired(): : if (samples[i] >= pct->bases[i].nextevt) 0.00 : ffff8000101761c8: cmp x3, x5 0.00 : ffff8000101761cc: ccmp x2, x4, #0x0, hi // hi = pmore : store_samples(): : samples[CPUCLOCK_PROF] = stime + utime; 0.00 : ffff8000101761d0: add x0, x4, x0 : task_cputimers_expired(): : if (samples[i] >= pct->bases[i].nextevt) 0.00 : ffff8000101761d4: ccmp x1, x0, #0x0, hi // hi = pmore 0.00 : ffff8000101761d8: b.ls ffff800010176258 // b.plast : fastpath_timer_check(): : sig = tsk->signal; 22.59 : ffff8000101761dc: ldr x0, [x19, #1640] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101761e0: ldr w1, [x0, #344] : fastpath_timer_check(): : if (READ_ONCE(pct->timers_active) && !READ_ONCE(pct->expiry_active)) { 0.00 : ffff8000101761e4: cbnz w1, ffff800010176218 : dl_prio(): : : #define MAX_DL_PRIO 0 : : static inline int dl_prio(int prio) : { : if (unlikely(prio < MAX_DL_PRIO)) 0.00 : ffff8000101761e8: ldr w0, [x19, #108] 0.00 : ffff8000101761ec: tbnz w0, #31, ffff800010176398 : run_posix_cpu_timers(): : */ : if (likely(cpu_firing >= 0)) : cpu_timer_fire(timer); : spin_unlock(&timer->it_lock); : } : } 11.31 : ffff8000101761f0: add x21, x21, #0x908 0.00 : ffff8000101761f4: ldr x1, [x29, #152] 5.41 : ffff8000101761f8: ldr x0, [x21] 0.00 : ffff8000101761fc: eor x0, x1, x0 0.00 : ffff800010176200: cbnz x0, ffff800010176568 0.00 : ffff800010176204: ldp x19, x20, [sp, #16] 10.51 : ffff800010176208: ldr x21, [sp, #32] 0.00 : ffff80001017620c: ldr x23, [sp, #48] 0.00 : ffff800010176210: ldp x29, x30, [sp], #160 0.00 : ffff800010176214: ret : __read_once_size(): 0.00 : ffff800010176218: ldr w1, [x0, #348] : fastpath_timer_check(): : if (READ_ONCE(pct->timers_active) && !READ_ONCE(pct->expiry_active)) { 0.00 : ffff80001017621c: cbnz w1, ffff8000101761e8 : __read_once_size(): 0.00 : ffff800010176220: ldr x2, [x0, #248] : task_cputimers_expired(): : if (samples[i] >= pct->bases[i].nextevt) 0.00 : ffff800010176224: add x0, x0, #0x110 : __read_once_size(): 0.00 : ffff800010176228: ldur x1, [x0, #-16] 0.00 : ffff80001017622c: ldur x4, [x0, #-8] : task_cputimers_expired(): 0.00 : ffff800010176230: ldr x3, [x0] : store_samples(): : samples[CPUCLOCK_PROF] = stime + utime; 0.00 : ffff800010176234: add x1, x2, x1 : task_cputimers_expired(): : if (samples[i] >= pct->bases[i].nextevt) 0.00 : ffff800010176238: cmp x1, x3 0.00 : ffff80001017623c: b.cs ffff800010176258 // b.hs, b.nlast 0.00 : ffff800010176240: ldr x1, [x0, #24] 0.00 : ffff800010176244: cmp x2, x1 0.00 : ffff800010176248: b.cs ffff800010176258 // b.hs, b.nlast 0.00 : ffff80001017624c: ldr x0, [x0, #48] 0.00 : ffff800010176250: cmp x4, x0 0.00 : ffff800010176254: b.cc ffff8000101761e8 // b.lo, b.ul, b.last : lock_task_sighand(): : static inline struct sighand_struct *lock_task_sighand(struct task_struct *task, : unsigned long *flags) : { : struct sighand_struct *ret; : : ret = __lock_task_sighand(task, flags); 0.00 : ffff800010176258: add x1, x29, #0x68 0.00 : ffff80001017625c: mov x0, x19 0.00 : ffff800010176260: bl ffff8000100f6aa0 <__lock_task_sighand> : run_posix_cpu_timers(): : if (!lock_task_sighand(tsk, &flags)) 0.00 : ffff800010176264: cbz x0, ffff8000101761f0 0.00 : ffff800010176268: str x22, [x29, #40] 0.00 : ffff80001017626c: str x24, [x29, #56] : dl_prio(): 0.00 : ffff800010176270: ldr w0, [x19, #108] 0.00 : ffff800010176274: tbnz w0, #31, ffff8000101764dc : expiry_cache_is_inactive(): : return !(~pct->bases[CPUCLOCK_PROF].nextevt | 0.00 : ffff800010176278: ldr x0, [x19, #1456] 0.00 : ffff80001017627c: ldr x2, [x20, #24] : ~pct->bases[CPUCLOCK_VIRT].nextevt | 0.00 : ffff800010176280: ldr x1, [x20, #48] : return !(~pct->bases[CPUCLOCK_PROF].nextevt | 0.00 : ffff800010176284: and x0, x0, x2 : ~pct->bases[CPUCLOCK_VIRT].nextevt | 0.00 : ffff800010176288: and x0, x0, x1 : check_thread_timers(): : if (expiry_cache_is_inactive(pct)) 0.00 : ffff80001017628c: cmn x0, #0x1 0.00 : ffff800010176290: b.eq ffff8000101762f8 // b.none : task_sample_cputime(): : store_samples(samples, stime, utime, p->se.sum_exec_runtime); 0.00 : ffff800010176294: ldr x3, [x19, #272] : check_thread_timers(): : collect_posix_cputimers(pct, samples, firing); 0.00 : ffff800010176298: mov x2, x23 : task_cputime(): 0.00 : ffff80001017629c: ldr x1, [x19, #1360] : store_samples(): : samples[CPUCLOCK_PROF] = stime + utime; 0.00 : ffff8000101762a0: ldr x0, [x19, #1368] : samples[CPUCLOCK_SCHED] = rtime; 0.00 : ffff8000101762a4: str x3, [x29, #144] : samples[CPUCLOCK_PROF] = stime + utime; 0.00 : ffff8000101762a8: add x0, x0, x1 : samples[CPUCLOCK_VIRT] = utime; 0.00 : ffff8000101762ac: stp x0, x1, [x29, #128] : check_thread_timers(): : collect_posix_cputimers(pct, samples, firing); 0.00 : ffff8000101762b0: mov x0, x20 0.00 : ffff8000101762b4: add x1, x29, #0x80 0.00 : ffff8000101762b8: bl ffff800010174e38 : task_rlimit(): : } : : static inline unsigned long task_rlimit(const struct task_struct *task, : unsigned int limit) : { : return READ_ONCE(task->signal->rlim[limit].rlim_cur); 0.00 : ffff8000101762bc: ldr x20, [x19, #1640] : __read_once_size(): 0.00 : ffff8000101762c0: ldr x22, [x20, #912] : check_thread_timers(): : if (soft != RLIM_INFINITY) { 0.00 : ffff8000101762c4: cmn x22, #0x1 0.00 : ffff8000101762c8: b.eq ffff8000101762fc // b.none : unsigned long rttime = tsk->rt.timeout * (USEC_PER_SEC / HZ); 0.00 : ffff8000101762cc: ldr x0, [x19, #464] : __read_once_size(): 0.00 : ffff8000101762d0: ldr x1, [x20, #920] : check_thread_timers(): 0.00 : ffff8000101762d4: lsl x20, x0, #5 0.00 : ffff8000101762d8: sub x20, x20, x0 0.00 : ffff8000101762dc: add x20, x0, x20, lsl #2 0.00 : ffff8000101762e0: lsl x20, x20, #5 : check_rlimit(): : if (time < limit) 0.00 : ffff8000101762e4: cmp x20, x1 0.00 : ffff8000101762e8: ccmn x1, #0x1, #0x4, cs // cs = hs, nlast 0.00 : ffff8000101762ec: b.ne ffff8000101763d4 // b.any 0.00 : ffff8000101762f0: cmp x22, x20 0.00 : ffff8000101762f4: b.ls ffff8000101763a4 // b.plast 0.00 : ffff8000101762f8: ldr x20, [x19, #1640] : __read_once_size(): 0.00 : ffff8000101762fc: ldr w0, [x20, #344] : check_process_timers(): : if (!READ_ONCE(pct->timers_active) || pct->expiry_active) 0.00 : ffff800010176300: cbz w0, ffff800010176310 0.00 : ffff800010176304: add x22, x20, #0x110 0.00 : ffff800010176308: ldr w0, [x22, #76] 0.00 : ffff80001017630c: cbz w0, ffff8000101763f0 : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff800010176310: ldr x0, [x19, #1648] 0.00 : ffff800010176314: ldr x1, [x29, #104] 0.00 : ffff800010176318: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : run_posix_cpu_timers(): : list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) { 0.00 : ffff80001017631c: ldr x19, [x29, #112] 0.00 : ffff800010176320: mov x20, x19 0.00 : ffff800010176324: cmp x19, x23 0.00 : ffff800010176328: ldr x22, [x19], #-168 0.00 : ffff80001017632c: sub x22, x22, #0xa8 0.00 : ffff800010176330: b.eq ffff80001017638c // b.none 0.00 : ffff800010176334: nop : spin_lock(): : raw_spin_lock(&lock->rlock); 0.00 : ffff800010176338: add x24, x19, #0x20 0.00 : ffff80001017633c: mov x0, x24 0.00 : ffff800010176340: bl ffff800010cd80f0 <_raw_spin_lock> : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff800010176344: ldp x1, x0, [x19, #168] : __list_del(): : next->prev = prev; 0.00 : ffff800010176348: str x0, [x1, #8] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001017634c: str x1, [x0] 0.00 : ffff800010176350: str x20, [x19, #168] : run_posix_cpu_timers(): : cpu_firing = timer->it.cpu.firing; 0.00 : ffff800010176354: ldr w0, [x19, #184] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010176358: str x20, [x19, #176] : run_posix_cpu_timers(): : timer->it.cpu.firing = 0; 0.00 : ffff80001017635c: str wzr, [x19, #184] : if (likely(cpu_firing >= 0)) 0.00 : ffff800010176360: tbnz w0, #31, ffff80001017636c : cpu_timer_fire(timer); 0.00 : ffff800010176364: mov x0, x19 0.00 : ffff800010176368: bl ffff800010175820 : list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) { 0.00 : ffff80001017636c: mov x20, x22 : spin_unlock(): : raw_spin_unlock(&lock->rlock); 0.00 : ffff800010176370: mov x0, x24 0.00 : ffff800010176374: mov x19, x22 0.00 : ffff800010176378: bl ffff800010cd7c38 <_raw_spin_unlock> : run_posix_cpu_timers(): 0.00 : ffff80001017637c: ldr x22, [x20, #168]! 0.00 : ffff800010176380: sub x22, x22, #0xa8 0.00 : ffff800010176384: cmp x20, x23 0.00 : ffff800010176388: b.ne ffff800010176338 // b.any 0.00 : ffff80001017638c: ldr x22, [x29, #40] 0.00 : ffff800010176390: ldr x24, [x29, #56] 0.00 : ffff800010176394: b ffff8000101761f0 : fastpath_timer_check(): : if (dl_task(tsk) && tsk->dl.dl_overrun) 0.00 : ffff800010176398: ldrb w0, [x19, #588] 0.00 : ffff80001017639c: tbz w0, #4, ffff8000101761f0 0.00 : ffff8000101763a0: b ffff800010176258 : check_rlimit(): 0.00 : ffff8000101763a4: mov w2, #0x0 // #0 0.00 : ffff8000101763a8: mov w1, #0x1 // #1 0.00 : ffff8000101763ac: mov w0, #0x18 // #24 0.00 : ffff8000101763b0: bl ffff800010175008 : check_thread_timers(): : if (check_rlimit(rttime, soft, SIGXCPU, true, false)) { 0.00 : ffff8000101763b4: tst w0, #0xff 0.00 : ffff8000101763b8: b.eq ffff8000101762f8 // b.none : tsk->signal->rlim[RLIMIT_RTTIME].rlim_cur = soft; 0.00 : ffff8000101763bc: ldr x0, [x19, #1640] : soft += USEC_PER_SEC; 0.00 : ffff8000101763c0: add x22, x22, #0xf4, lsl #12 0.00 : ffff8000101763c4: add x22, x22, #0x240 : tsk->signal->rlim[RLIMIT_RTTIME].rlim_cur = soft; 0.00 : ffff8000101763c8: str x22, [x0, #912] 0.00 : ffff8000101763cc: ldr x20, [x19, #1640] 0.00 : ffff8000101763d0: b ffff8000101762fc : check_rlimit(): 0.00 : ffff8000101763d4: mov w2, #0x1 // #1 0.00 : ffff8000101763d8: mov w0, #0x9 // #9 0.00 : ffff8000101763dc: mov w1, w2 0.00 : ffff8000101763e0: bl ffff800010175008 : check_thread_timers(): : if (hard != RLIM_INFINITY && 0.00 : ffff8000101763e4: tst w0, #0xff 0.00 : ffff8000101763e8: b.ne ffff8000101762f8 // b.any 0.00 : ffff8000101763ec: b ffff8000101762f0 0.00 : ffff8000101763f0: str x26, [x29, #72] : check_process_timers(): : pct->expiry_active = true; 0.00 : ffff8000101763f4: mov w26, #0x1 // #1 0.00 : ffff8000101763f8: str w26, [x22, #76] : collect_posix_cputimers(pct, samples, firing); 0.00 : ffff8000101763fc: mov x2, x23 0.00 : ffff800010176400: add x1, x29, #0x80 0.00 : ffff800010176404: mov x0, x22 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010176408: ldr x4, [x20, #248] 0.00 : ffff80001017640c: ldr x3, [x20, #256] 0.00 : ffff800010176410: ldr x5, [x20, #264] : store_samples(): : samples[CPUCLOCK_PROF] = stime + utime; 0.00 : ffff800010176414: add x3, x4, x3 : samples[CPUCLOCK_VIRT] = utime; 0.00 : ffff800010176418: stp x3, x4, [x29, #128] : samples[CPUCLOCK_SCHED] = rtime; 0.00 : ffff80001017641c: str x5, [x29, #144] : check_process_timers(): : collect_posix_cputimers(pct, samples, firing); 0.00 : ffff800010176420: bl ffff800010174e38 : check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], 0.00 : ffff800010176424: ldr x3, [x29, #128] 0.00 : ffff800010176428: mov w4, #0x1b // #27 0.00 : ffff80001017642c: mov x2, x22 0.00 : ffff800010176430: add x1, x20, #0xd8 0.00 : ffff800010176434: mov x0, x19 0.00 : ffff800010176438: bl ffff800010174f88 : check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], 0.00 : ffff80001017643c: ldr x3, [x29, #136] 0.00 : ffff800010176440: mov x0, x19 0.00 : ffff800010176444: mov w4, #0x1a // #26 0.00 : ffff800010176448: add x2, x20, #0x128 0.00 : ffff80001017644c: add x1, x20, #0xe8 0.00 : ffff800010176450: bl ffff800010174f88 : task_rlimit(): 0.00 : ffff800010176454: ldr x0, [x19, #1640] : __read_once_size(): 0.00 : ffff800010176458: ldr x24, [x0, #672] : check_process_timers(): : if (soft != RLIM_INFINITY) { 0.00 : ffff80001017645c: cmn x24, #0x1 0.00 : ffff800010176460: b.eq ffff800010176500 // b.none 0.00 : ffff800010176464: str x25, [x29, #64] 0.00 : ffff800010176468: str x27, [x29, #80] : u64 ptime = samples[CPUCLOCK_PROF]; 0.00 : ffff80001017646c: ldr x27, [x29, #128] : __read_once_size(): 0.00 : ffff800010176470: ldr x1, [x0, #680] : check_process_timers(): : u64 softns = (u64)soft * NSEC_PER_SEC; 0.00 : ffff800010176474: mov x0, #0xca00 // #51712 0.00 : ffff800010176478: movk x0, #0x3b9a, lsl #16 0.00 : ffff80001017647c: mul x25, x24, x0 : u64 hardns = (u64)hard * NSEC_PER_SEC; 0.00 : ffff800010176480: mul x0, x1, x0 : check_rlimit(): : if (time < limit) 0.00 : ffff800010176484: cmp x27, x0 0.00 : ffff800010176488: ccmn x1, #0x1, #0x4, cs // cs = hs, nlast 0.00 : ffff80001017648c: b.ne ffff800010176544 // b.any 0.00 : ffff800010176490: cmp x27, x25 0.00 : ffff800010176494: b.cs ffff800010176514 // b.hs, b.nlast : check_process_timers(): : if (softns < pct->bases[CPUCLOCK_PROF].nextevt) 0.00 : ffff800010176498: ldr x1, [x20, #272] 0.00 : ffff80001017649c: cmp x1, x25 0.00 : ffff8000101764a0: b.ls ffff800010176508 // b.plast : pct->bases[CPUCLOCK_PROF].nextevt = softns; 0.00 : ffff8000101764a4: str x25, [x20, #272] 0.00 : ffff8000101764a8: mov x1, x25 0.00 : ffff8000101764ac: ldr x25, [x29, #64] 0.00 : ffff8000101764b0: ldr x27, [x29, #80] : expiry_cache_is_inactive(): : ~pct->bases[CPUCLOCK_VIRT].nextevt | 0.00 : ffff8000101764b4: ldr x0, [x22, #24] 0.00 : ffff8000101764b8: ldr x2, [x22, #48] 0.00 : ffff8000101764bc: and x0, x0, x2 0.00 : ffff8000101764c0: and x0, x0, x1 : check_process_timers(): : if (expiry_cache_is_inactive(pct)) 0.00 : ffff8000101764c4: cmn x0, #0x1 0.00 : ffff8000101764c8: b.ne ffff8000101764d0 // b.any : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101764cc: str wzr, [x20, #344] : check_process_timers(): : pct->expiry_active = false; 0.00 : ffff8000101764d0: str wzr, [x22, #76] 0.00 : ffff8000101764d4: ldr x26, [x29, #72] 0.00 : ffff8000101764d8: b ffff800010176310 : check_dl_overrun(): : if (tsk->dl.dl_overrun) { 0.00 : ffff8000101764dc: ldrb w0, [x19, #588] 0.00 : ffff8000101764e0: tbz w0, #4, ffff800010176278 : tsk->dl.dl_overrun = 0; 0.00 : ffff8000101764e4: and w0, w0, #0xffffffef 0.00 : ffff8000101764e8: strb w0, [x19, #588] : __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk); 0.00 : ffff8000101764ec: mov x2, x19 0.00 : ffff8000101764f0: mov x1, #0x1 // #1 0.00 : ffff8000101764f4: mov w0, #0x18 // #24 0.00 : ffff8000101764f8: bl ffff8000100f7978 <__group_send_sig_info> 0.00 : ffff8000101764fc: b ffff800010176278 0.00 : ffff800010176500: ldr x1, [x22] 0.00 : ffff800010176504: b ffff8000101764b4 0.00 : ffff800010176508: ldr x25, [x29, #64] 0.00 : ffff80001017650c: ldr x27, [x29, #80] 0.00 : ffff800010176510: b ffff8000101764b4 : check_rlimit(): 0.00 : ffff800010176514: mov w2, #0x0 // #0 0.00 : ffff800010176518: mov w1, #0x0 // #0 0.00 : ffff80001017651c: mov w0, #0x18 // #24 0.00 : ffff800010176520: bl ffff800010175008 : check_process_timers(): : if (check_rlimit(ptime, softns, SIGXCPU, false, false)) { 0.00 : ffff800010176524: tst w0, #0xff 0.00 : ffff800010176528: b.eq ffff800010176498 // b.none : softns += NSEC_PER_SEC; 0.00 : ffff80001017652c: mov x0, #0xca00 // #51712 : sig->rlim[RLIMIT_CPU].rlim_cur = soft + 1; 0.00 : ffff800010176530: add x24, x24, #0x1 : softns += NSEC_PER_SEC; 0.00 : ffff800010176534: movk x0, #0x3b9a, lsl #16 : sig->rlim[RLIMIT_CPU].rlim_cur = soft + 1; 0.00 : ffff800010176538: str x24, [x20, #672] : softns += NSEC_PER_SEC; 0.00 : ffff80001017653c: add x25, x25, x0 0.00 : ffff800010176540: b ffff800010176498 : check_rlimit(): 0.00 : ffff800010176544: mov w2, w26 0.00 : ffff800010176548: mov w1, #0x0 // #0 0.00 : ffff80001017654c: mov w0, #0x9 // #9 0.00 : ffff800010176550: bl ffff800010175008 : check_process_timers(): : if (hard != RLIM_INFINITY && 0.00 : ffff800010176554: tst w0, #0xff 0.00 : ffff800010176558: b.eq ffff800010176490 // b.none 0.00 : ffff80001017655c: ldp x25, x26, [x29, #64] 0.00 : ffff800010176560: ldr x27, [x29, #80] 0.00 : ffff800010176564: b ffff800010176310 0.00 : ffff800010176568: str x22, [x29, #40] 0.00 : ffff80001017656c: stp x24, x25, [x29, #56] 0.00 : ffff800010176570: stp x26, x27, [x29, #72] : run_posix_cpu_timers(): : } 0.00 : ffff800010176574: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (71 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010461418 <__blk_queue_split>: : __blk_queue_split(): : * of the caller to ensure that @q is only released after processing of the : * split bio has finished. : */ : void __blk_queue_split(struct request_queue *q, struct bio **bio, : unsigned int *nr_segs) : { 0.00 : ffff800010461418: stp x29, x30, [sp, #-144]! 0.00 : ffff80001046141c: mov x29, sp 29.61 : ffff800010461420: stp x19, x20, [sp, #16] 0.00 : ffff800010461424: mov x20, x1 2.86 : ffff800010461428: str x21, [sp, #32] 0.00 : ffff80001046142c: adrp x19, ffff800011909000 6.89 : ffff800010461430: stp x25, x26, [sp, #64] 0.00 : ffff800010461434: add x1, x19, #0x908 0.00 : ffff800010461438: mov x26, x0 0.00 : ffff80001046143c: mov x21, x2 : struct bio *split = NULL; : : switch (bio_op(*bio)) { 0.00 : ffff800010461440: ldr x25, [x20] : { 0.00 : ffff800010461444: ldr x0, [x1] 4.30 : ffff800010461448: str x0, [x29, #136] 0.00 : ffff80001046144c: mov x0, #0x0 // #0 : switch (bio_op(*bio)) { 0.00 : ffff800010461450: ldrb w0, [x25, #16] 0.00 : ffff800010461454: cmp w0, #0x5 0.00 : ffff800010461458: b.eq ffff800010461610 <__blk_queue_split+0x1f8> // b.none 7.13 : ffff80001046145c: b.ls ffff8000104614d0 <__blk_queue_split+0xb8> // b.plast 0.00 : ffff800010461460: cmp w0, #0x7 0.00 : ffff800010461464: b.eq ffff8000104614b0 <__blk_queue_split+0x98> // b.none 0.00 : ffff800010461468: cmp w0, #0x9 0.00 : ffff80001046146c: b.ne ffff8000104614d8 <__blk_queue_split+0xc0> // b.any : blk_bio_write_zeroes_split(): : *nsegs = 0; 0.00 : ffff800010461470: str wzr, [x2] : if (!q->limits.max_write_zeroes_sectors) 0.00 : ffff800010461474: ldr w1, [x26, #1116] 0.00 : ffff800010461478: cbz w1, ffff800010461488 <__blk_queue_split+0x70> : blk_bio_write_same_split(): : if (bio_sectors(bio) <= q->limits.max_write_same_sectors) 0.00 : ffff80001046147c: ldr w0, [x25, #40] 0.00 : ffff800010461480: cmp w1, w0, lsr #9 0.00 : ffff800010461484: b.cc ffff8000104616bc <__blk_queue_split+0x2a4> // b.lo, b.ul, b.last : __blk_queue_split(): : bio_chain(split, *bio); : trace_block_split(q, split, (*bio)->bi_iter.bi_sector); : generic_make_request(*bio); : *bio = split; : } : } 1.41 : ffff800010461488: add x19, x19, #0x908 0.00 : ffff80001046148c: ldr x1, [x29, #136] 18.30 : ffff800010461490: ldr x0, [x19] 0.00 : ffff800010461494: eor x0, x1, x0 0.00 : ffff800010461498: cbnz x0, ffff800010461864 <__blk_queue_split+0x44c> 15.49 : ffff80001046149c: ldp x19, x20, [sp, #16] 6.88 : ffff8000104614a0: ldr x21, [sp, #32] 0.00 : ffff8000104614a4: ldp x25, x26, [sp, #64] 0.00 : ffff8000104614a8: ldp x29, x30, [sp], #144 0.00 : ffff8000104614ac: ret : blk_bio_write_same_split(): : *nsegs = 1; 0.00 : ffff8000104614b0: mov w0, #0x1 // #1 0.00 : ffff8000104614b4: str w0, [x2] : if (!q->limits.max_write_same_sectors) 0.00 : ffff8000104614b8: ldr w1, [x26, #1112] 0.00 : ffff8000104614bc: cbz w1, ffff800010461488 <__blk_queue_split+0x70> : if (bio_sectors(bio) <= q->limits.max_write_same_sectors) 0.00 : ffff8000104614c0: ldr w0, [x25, #40] 0.00 : ffff8000104614c4: cmp w1, w0, lsr #9 0.00 : ffff8000104614c8: b.cs ffff800010461488 <__blk_queue_split+0x70> // b.hs, b.nlast 0.00 : ffff8000104614cc: b ffff8000104616bc <__blk_queue_split+0x2a4> : __blk_queue_split(): : switch (bio_op(*bio)) { 1.43 : ffff8000104614d0: cmp w0, #0x3 0.00 : ffff8000104614d4: b.eq ffff800010461610 <__blk_queue_split+0x1f8> // b.none : if (!q->limits.chunk_sectors && 4.26 : ffff8000104614d8: ldr w0, [x26, #1072] 0.00 : ffff8000104614dc: cbnz w0, ffff8000104616d4 <__blk_queue_split+0x2bc> 0.00 : ffff8000104614e0: ldrh w0, [x25, #96] 0.00 : ffff8000104614e4: cmp w0, #0x1 0.00 : ffff8000104614e8: b.eq ffff800010461848 <__blk_queue_split+0x430> // b.none 0.00 : ffff8000104614ec: stp x22, x23, [x29, #40] : split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs); 0.00 : ffff8000104614f0: add x18, x26, #0x610 0.00 : ffff8000104614f4: str x24, [x29, #56] 0.00 : ffff8000104614f8: stp x27, x28, [x29, #80] : blk_bio_segment_split(): : unsigned nsegs = 0, sectors = 0; 0.00 : ffff8000104614fc: stp wzr, wzr, [x29, #96] 0.00 : ffff800010461500: ldr w4, [x25, #32] : get_max_io_size(): : unsigned sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector); 0.00 : ffff800010461504: ldr w1, [x26, #1076] : unsigned pbs = queue_physical_block_size(q) >> SECTOR_SHIFT; 0.00 : ffff800010461508: ldr w0, [x26, #1084] : queue_logical_block_size(): : : static inline unsigned queue_logical_block_size(const struct request_queue *q) : { : int retval = 512; : : if (q && q->limits.logical_block_size) 0.00 : ffff80001046150c: mov w2, #0x1 // #1 : get_max_io_size(): 0.00 : ffff800010461510: lsr w0, w0, #9 : queue_logical_block_size(): 0.00 : ffff800010461514: cbz x26, ffff800010461528 <__blk_queue_split+0x110> 0.00 : ffff800010461518: ldr w3, [x26, #1088] 0.00 : ffff80001046151c: cmp w3, #0x0 0.00 : ffff800010461520: lsr w3, w3, #9 0.00 : ffff800010461524: csel w2, w3, w2, ne // ne = any : get_max_io_size(): : unsigned start_offset = bio->bi_iter.bi_sector & (pbs - 1); 0.00 : ffff800010461528: sub w3, w0, #0x1 : max_sectors &= ~(pbs - 1); 0.00 : ffff80001046152c: neg w0, w0 : unsigned start_offset = bio->bi_iter.bi_sector & (pbs - 1); 0.00 : ffff800010461530: and w3, w3, w4 : return max_sectors - start_offset; 0.00 : ffff800010461534: sub w2, w2, #0x1 : max_sectors += start_offset; 0.00 : ffff800010461538: add w4, w3, w1 : blk_bio_segment_split(): : const unsigned max_segs = queue_max_segments(q); 0.00 : ffff80001046153c: ldrh w27, [x26, #1128] : get_max_io_size(): : max_sectors &= ~(pbs - 1); 0.00 : ffff800010461540: and w0, w0, w4 : return max_sectors - start_offset; 0.00 : ffff800010461544: and w1, w2, w1 0.00 : ffff800010461548: sub w14, w0, w3 0.00 : ffff80001046154c: cmp w3, w0 : blk_bio_segment_split(): : bio_for_each_bvec(bv, bio, iter) { 0.00 : ffff800010461550: ldp w24, w23, [x25, #40] : get_max_io_size(): : return max_sectors - start_offset; 0.00 : ffff800010461554: csel w14, w1, w14, cs // cs = hs, nlast : blk_bio_segment_split(): : bio_for_each_bvec(bv, bio, iter) { 0.00 : ffff800010461558: ldr w22, [x25, #48] 0.00 : ffff80001046155c: mov w2, #0x0 // #0 : struct bio_vec bv, bvprv, *bvprvp = NULL; 0.00 : ffff800010461560: mov x3, #0x0 // #0 : bio_no_advance_iter(): : return false; : } : : static inline bool bio_no_advance_iter(struct bio *bio) : { : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff800010461564: mov w15, #0xfb // #251 : blk_bio_segment_split(): : bio_for_each_bvec(bv, bio, iter) { 0.00 : ffff800010461568: cbz w24, ffff80001046182c <__blk_queue_split+0x414> 0.00 : ffff80001046156c: ldr x0, [x25, #104] 0.00 : ffff800010461570: ubfiz x28, x23, #4, #32 0.00 : ffff800010461574: add x1, x0, x28 0.00 : ffff800010461578: ldr x0, [x0, x28] 0.00 : ffff80001046157c: str x0, [x29, #104] 0.00 : ffff800010461580: ldp w0, w1, [x1, #8] 0.00 : ffff800010461584: sub w0, w0, w22 0.00 : ffff800010461588: add w1, w22, w1 0.00 : ffff80001046158c: cmp w0, w24 0.00 : ffff800010461590: csel w0, w0, w24, ls // ls = plast 0.00 : ffff800010461594: stp w0, w1, [x29, #112] : if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset)) 0.00 : ffff800010461598: cbz x3, ffff800010461720 <__blk_queue_split+0x308> : bvec_gap_to_prev(): : * the SG list. Most drivers don't care about this, but some do. : */ : static inline bool bvec_gap_to_prev(struct request_queue *q, : struct bio_vec *bprv, unsigned int offset) : { : if (!queue_virt_boundary(q)) 0.00 : ffff80001046159c: ldr x4, [x26, #1056] 0.00 : ffff8000104615a0: cbz x4, ffff800010461720 <__blk_queue_split+0x308> : __bvec_gap_to_prev(): : return (offset & queue_virt_boundary(q)) || 0.00 : ffff8000104615a4: tst w1, w4 0.00 : ffff8000104615a8: b.eq ffff80001046170c <__blk_queue_split+0x2f4> // b.none : blk_bio_segment_split(): : return bio_split(bio, sectors, GFP_NOIO, bs); 0.00 : ffff8000104615ac: ldr w1, [x29, #100] 0.00 : ffff8000104615b0: mov x3, x18 : *segs = nsegs; 0.00 : ffff8000104615b4: str w2, [x21] : return bio_split(bio, sectors, GFP_NOIO, bs); 0.00 : ffff8000104615b8: mov x0, x25 0.00 : ffff8000104615bc: mov w2, #0xc00 // #3072 0.00 : ffff8000104615c0: bl ffff800010457260 0.00 : ffff8000104615c4: ldr x24, [x29, #56] 0.00 : ffff8000104615c8: ldp x22, x23, [x29, #40] 0.00 : ffff8000104615cc: mov x21, x0 0.00 : ffff8000104615d0: ldp x27, x28, [x29, #80] : __blk_queue_split(): : if (split) { 0.00 : ffff8000104615d4: cbz x21, ffff800010461488 <__blk_queue_split+0x70> : split->bi_opf |= REQ_NOMERGE; 0.00 : ffff8000104615d8: ldr w1, [x21, #16] : bio_chain(split, *bio); 0.00 : ffff8000104615dc: mov x0, x21 : split->bi_opf |= REQ_NOMERGE; 0.00 : ffff8000104615e0: orr w1, w1, #0x4000 0.00 : ffff8000104615e4: str w1, [x21, #16] : bio_set_flag(*bio, BIO_QUEUE_ENTERED); 0.00 : ffff8000104615e8: ldr x2, [x20] : bio_set_flag(): : return (bio->bi_flags & (1U << bit)) != 0; : } : : static inline void bio_set_flag(struct bio *bio, unsigned int bit) : { : bio->bi_flags |= (1U << bit); 0.00 : ffff8000104615ec: ldrh w1, [x2, #20] 0.00 : ffff8000104615f0: orr w1, w1, #0x800 0.00 : ffff8000104615f4: strh w1, [x2, #20] : __blk_queue_split(): : bio_chain(split, *bio); 0.00 : ffff8000104615f8: ldr x1, [x20] 0.00 : ffff8000104615fc: bl ffff800010455df8 : generic_make_request(*bio); 0.00 : ffff800010461600: ldr x0, [x20] 0.00 : ffff800010461604: bl ffff80001045b4c0 : *bio = split; 0.00 : ffff800010461608: str x21, [x20] : } 0.00 : ffff80001046160c: b ffff800010461488 <__blk_queue_split+0x70> : blk_bio_discard_split(): : *nsegs = 1; 0.00 : ffff800010461610: mov w2, #0x1 // #1 0.00 : ffff800010461614: str w2, [x21] : __blk_queue_split(): : split = blk_bio_discard_split(q, *bio, &q->bio_split, nr_segs); 0.00 : ffff800010461618: add x3, x26, #0x610 : queue_logical_block_size(): 0.00 : ffff80001046161c: mov w0, #0x7fffff // #8388607 : blk_bio_discard_split(): : granularity = max(q->limits.discard_granularity >> 9, 1U); 0.00 : ffff800010461620: ldr w4, [x26, #1120] : max_discard_sectors = min(q->limits.max_discard_sectors, 0.00 : ffff800010461624: ldr w1, [x26, #1104] : granularity = max(q->limits.discard_granularity >> 9, 1U); 0.00 : ffff800010461628: lsr w4, w4, #9 0.00 : ffff80001046162c: cmp w4, #0x0 0.00 : ffff800010461630: csel w4, w4, w2, ne // ne = any : queue_logical_block_size(): 0.00 : ffff800010461634: cbz x26, ffff80001046164c <__blk_queue_split+0x234> 0.00 : ffff800010461638: ldr w5, [x26, #1088] 0.00 : ffff80001046163c: neg w2, w5 0.00 : ffff800010461640: cmp w5, #0x0 0.00 : ffff800010461644: lsr w2, w2, #9 0.00 : ffff800010461648: csel w0, w2, w0, ne // ne = any : blk_bio_discard_split(): : max_discard_sectors = min(q->limits.max_discard_sectors, 0.00 : ffff80001046164c: cmp w1, w0 0.00 : ffff800010461650: csel w1, w1, w0, ls // ls = plast : max_discard_sectors -= max_discard_sectors % granularity; 0.00 : ffff800010461654: udiv w0, w1, w4 0.00 : ffff800010461658: msub w0, w0, w4, w1 : if (unlikely(!max_discard_sectors)) { 0.00 : ffff80001046165c: subs w1, w1, w0 0.00 : ffff800010461660: b.eq ffff800010461488 <__blk_queue_split+0x70> // b.none : if (bio_sectors(bio) <= max_discard_sectors) 0.00 : ffff800010461664: ldr w0, [x25, #40] 0.00 : ffff800010461668: cmp w1, w0, lsr #9 0.00 : ffff80001046166c: b.cs ffff800010461488 <__blk_queue_split+0x70> // b.hs, b.nlast : alignment = (q->limits.discard_alignment >> 9) % granularity; 0.00 : ffff800010461670: ldr w6, [x26, #1124] : tmp = sector_div(tmp, granularity); 0.00 : ffff800010461674: mov w9, w4 : tmp = bio->bi_iter.bi_sector + split_sectors - alignment; 0.00 : ffff800010461678: ldr x5, [x25, #32] 0.00 : ffff80001046167c: mov w8, w1 : return bio_split(bio, split_sectors, GFP_NOIO, bs); 0.00 : ffff800010461680: mov w2, #0xc00 // #3072 0.00 : ffff800010461684: mov x0, x25 : alignment = (q->limits.discard_alignment >> 9) % granularity; 0.00 : ffff800010461688: lsr w6, w6, #9 0.00 : ffff80001046168c: udiv w7, w6, w4 : tmp = bio->bi_iter.bi_sector + split_sectors - alignment; 0.00 : ffff800010461690: msub w4, w7, w4, w6 0.00 : ffff800010461694: sub x4, x5, x4 0.00 : ffff800010461698: add x5, x4, x8 : tmp = sector_div(tmp, granularity); 0.00 : ffff80001046169c: udiv x4, x5, x9 0.00 : ffff8000104616a0: msub x4, x4, x9, x5 : split_sectors -= tmp; 0.00 : ffff8000104616a4: cmp x8, x4 0.00 : ffff8000104616a8: sub w4, w1, w4 : return bio_split(bio, split_sectors, GFP_NOIO, bs); 0.00 : ffff8000104616ac: csel w1, w4, w1, hi // hi = pmore 0.00 : ffff8000104616b0: bl ffff800010457260 0.00 : ffff8000104616b4: mov x21, x0 0.00 : ffff8000104616b8: b ffff8000104615d4 <__blk_queue_split+0x1bc> : blk_bio_write_same_split(): : return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs); 0.00 : ffff8000104616bc: add x3, x26, #0x610 0.00 : ffff8000104616c0: mov w2, #0xc00 // #3072 0.00 : ffff8000104616c4: mov x0, x25 0.00 : ffff8000104616c8: bl ffff800010457260 0.00 : ffff8000104616cc: mov x21, x0 : __blk_queue_split(): : break; 0.00 : ffff8000104616d0: b ffff8000104615d4 <__blk_queue_split+0x1bc> 0.00 : ffff8000104616d4: stp x22, x23, [x29, #40] : blk_max_size_offset(): : return min(q->limits.max_sectors, (unsigned int)(q->limits.chunk_sectors - 0.00 : ffff8000104616d8: sub w1, w0, #0x1 0.00 : ffff8000104616dc: str x24, [x29, #56] : __blk_queue_split(): : split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs); 0.00 : ffff8000104616e0: add x18, x26, #0x610 0.00 : ffff8000104616e4: stp x27, x28, [x29, #80] : blk_bio_segment_split(): : unsigned nsegs = 0, sectors = 0; 0.00 : ffff8000104616e8: stp wzr, wzr, [x29, #96] : blk_max_size_offset(): 0.00 : ffff8000104616ec: ldr x3, [x25, #32] 0.00 : ffff8000104616f0: ldr w2, [x26, #1076] 0.00 : ffff8000104616f4: and w1, w1, w3 0.00 : ffff8000104616f8: mov w4, w3 0.00 : ffff8000104616fc: sub w0, w0, w1 0.00 : ffff800010461700: cmp w0, w2 0.00 : ffff800010461704: csel w1, w0, w2, ls // ls = plast 0.00 : ffff800010461708: b ffff800010461508 <__blk_queue_split+0xf0> : __bvec_gap_to_prev(): : ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); 0.00 : ffff80001046170c: ldp w5, w3, [x3, #8] 0.00 : ffff800010461710: add w3, w3, w5 : return (offset & queue_virt_boundary(q)) || 0.00 : ffff800010461714: tst x3, x4 0.00 : ffff800010461718: b.ne ffff8000104615ac <__blk_queue_split+0x194> // b.any 0.00 : ffff80001046171c: nop : blk_bio_segment_split(): : if (nsegs < max_segs && 0.00 : ffff800010461720: cmp w27, w2 0.00 : ffff800010461724: b.ls ffff800010461784 <__blk_queue_split+0x36c> // b.plast : sectors + (bv.bv_len >> 9) <= max_sectors && 0.00 : ffff800010461728: ldr w3, [x29, #100] 0.00 : ffff80001046172c: add w3, w3, w0, lsr #9 : if (nsegs < max_segs && 0.00 : ffff800010461730: cmp w3, w14 0.00 : ffff800010461734: b.hi ffff800010461784 <__blk_queue_split+0x36c> // b.pmore : bv.bv_offset + bv.bv_len <= PAGE_SIZE) { 0.00 : ffff800010461738: add w1, w1, w0 : sectors + (bv.bv_len >> 9) <= max_sectors && 0.00 : ffff80001046173c: cmp w1, #0x1, lsl #12 0.00 : ffff800010461740: b.hi ffff800010461784 <__blk_queue_split+0x36c> // b.pmore : nsegs++; 0.00 : ffff800010461744: add w2, w2, #0x1 : sectors += bv.bv_len >> 9; 0.00 : ffff800010461748: stp w2, w3, [x29, #96] 0.00 : ffff80001046174c: nop : bio_advance_iter(): : if (bio_no_advance_iter(bio)) 0.00 : ffff800010461750: ldr w1, [x25, #16] : blk_bio_segment_split(): : bvprv = bv; 0.00 : ffff800010461754: ldp x2, x3, [x29, #104] 0.00 : ffff800010461758: stp x2, x3, [x29, #120] : bio_no_advance_iter(): : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff80001046175c: and w2, w1, #0xff 0.00 : ffff800010461760: and w1, w1, w15 0.00 : ffff800010461764: sub w2, w2, #0x5 : bio_op(bio) == REQ_OP_WRITE_SAME || 0.00 : ffff800010461768: tst w2, #0xfffffffb 0.00 : ffff80001046176c: ccmp w1, #0x3, #0x4, ne // ne = any 0.00 : ffff800010461770: b.ne ffff8000104617b0 <__blk_queue_split+0x398> // b.any : bio_advance_iter(): : iter->bi_size -= bytes; 0.00 : ffff800010461774: sub w24, w24, w0 0.00 : ffff800010461778: add x3, x29, #0x78 0.00 : ffff80001046177c: ldr w2, [x29, #96] 0.00 : ffff800010461780: b ffff800010461568 <__blk_queue_split+0x150> : blk_bio_segment_split(): : } else if (bvec_split_segs(q, &bv, &nsegs, §ors, max_segs, 0.00 : ffff800010461784: mov w5, w14 0.00 : ffff800010461788: mov w4, w27 0.00 : ffff80001046178c: add x3, x29, #0x64 0.00 : ffff800010461790: add x2, x29, #0x60 0.00 : ffff800010461794: add x1, x29, #0x68 0.00 : ffff800010461798: mov x0, x26 0.00 : ffff80001046179c: bl ffff800010460cb8 0.00 : ffff8000104617a0: tst w0, #0xff 0.00 : ffff8000104617a4: b.ne ffff800010461840 <__blk_queue_split+0x428> // b.any 0.00 : ffff8000104617a8: ldr w0, [x29, #112] 0.00 : ffff8000104617ac: b ffff800010461750 <__blk_queue_split+0x338> : bvec_iter_advance(): : static inline bool bvec_iter_advance(const struct bio_vec *bv, : struct bvec_iter *iter, unsigned bytes) : { : unsigned int idx = iter->bi_idx; : : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff8000104617b0: cmp w0, w24 : bio_advance_iter(): : bvec_iter_advance(bio->bi_io_vec, iter, bytes); 0.00 : ffff8000104617b4: ldr x1, [x25, #104] : bvec_iter_advance(): 0.00 : ffff8000104617b8: b.hi ffff800010461800 <__blk_queue_split+0x3e8> // b.pmore : "Attempted to advance past end of bvec iter\n")) { : iter->bi_size = 0; : return false; : } : : iter->bi_size -= bytes; 0.00 : ffff8000104617bc: sub w24, w24, w0 : bytes += iter->bi_bvec_done; : : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff8000104617c0: adds w22, w22, w0 0.00 : ffff8000104617c4: b.eq ffff800010461778 <__blk_queue_split+0x360> // b.none 0.00 : ffff8000104617c8: add x28, x1, x28 0.00 : ffff8000104617cc: ldr w0, [x28, #8] 0.00 : ffff8000104617d0: cmp w0, w22 0.00 : ffff8000104617d4: b.ls ffff8000104617f0 <__blk_queue_split+0x3d8> // b.plast 0.00 : ffff8000104617d8: b ffff800010461778 <__blk_queue_split+0x360> 0.00 : ffff8000104617dc: ubfiz x0, x23, #4, #32 0.00 : ffff8000104617e0: add x0, x1, x0 0.00 : ffff8000104617e4: ldr w0, [x0, #8] 0.00 : ffff8000104617e8: cmp w0, w22 0.00 : ffff8000104617ec: b.hi ffff800010461778 <__blk_queue_split+0x360> // b.pmore : bytes -= bv[idx].bv_len; : idx++; 0.00 : ffff8000104617f0: add w23, w23, #0x1 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff8000104617f4: subs w22, w22, w0 0.00 : ffff8000104617f8: b.ne ffff8000104617dc <__blk_queue_split+0x3c4> // b.any 0.00 : ffff8000104617fc: b ffff800010461778 <__blk_queue_split+0x360> : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff800010461800: adrp x1, ffff800011ad8000 0.00 : ffff800010461804: ldrb w0, [x1, #1295] 0.00 : ffff800010461808: cbnz w0, ffff800010461828 <__blk_queue_split+0x410> 0.00 : ffff80001046180c: mov w2, #0x1 // #1 0.00 : ffff800010461810: adrp x0, ffff8000111e9000 0.00 : ffff800010461814: strb w2, [x1, #1295] 0.00 : ffff800010461818: add x0, x0, #0x680 0.00 : ffff80001046181c: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010461820: brk #0x800 0.00 : ffff800010461824: nop 0.00 : ffff800010461828: ldr w2, [x29, #96] : blk_bio_segment_split(): : *segs = nsegs; 0.00 : ffff80001046182c: str w2, [x21] 0.00 : ffff800010461830: ldp x22, x23, [x29, #40] 0.00 : ffff800010461834: ldr x24, [x29, #56] 0.00 : ffff800010461838: ldp x27, x28, [x29, #80] 0.00 : ffff80001046183c: b ffff800010461488 <__blk_queue_split+0x70> 0.00 : ffff800010461840: ldr w2, [x29, #96] 0.00 : ffff800010461844: b ffff8000104615ac <__blk_queue_split+0x194> : __blk_queue_split(): : ((*bio)->bi_io_vec[0].bv_len + 0.00 : ffff800010461848: ldr x2, [x25, #104] 1.44 : ffff80001046184c: ldp w1, w2, [x2, #8] 0.00 : ffff800010461850: add w1, w1, w2 : (*bio)->bi_vcnt == 1 && 0.00 : ffff800010461854: cmp w1, #0x1, lsl #12 0.00 : ffff800010461858: b.hi ffff8000104614ec <__blk_queue_split+0xd4> // b.pmore : *nr_segs = 1; 0.00 : ffff80001046185c: str w0, [x21] 0.00 : ffff800010461860: b ffff800010461488 <__blk_queue_split+0x70> 0.00 : ffff800010461864: stp x22, x23, [x29, #40] 0.00 : ffff800010461868: str x24, [x29, #56] 0.00 : ffff80001046186c: stp x27, x28, [x29, #80] : } 0.00 : ffff800010461870: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (50 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104b1298 <__sbitmap_queue_get>: : __sbitmap_queue_get(): : sbitmap_resize(&sbq->sb, depth); : } : EXPORT_SYMBOL_GPL(sbitmap_queue_resize); : : int __sbitmap_queue_get(struct sbitmap_queue *sbq) : { 0.00 : ffff8000104b1298: stp x29, x30, [sp, #-64]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 3.89 : ffff8000104b129c: mrs x1, sp_el0 : __sbitmap_queue_get(): 0.00 : ffff8000104b12a0: mov x29, sp 0.00 : ffff8000104b12a4: stp x19, x20, [sp, #16] 0.00 : ffff8000104b12a8: mov x19, x0 0.00 : ffff8000104b12ac: str x21, [sp, #32] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 6.12 : ffff8000104b12b0: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000104b12b4: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 3.86 : ffff8000104b12b8: str w2, [x1, #16] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000104b12bc: mrs x2, tpidr_el1 : __sbitmap_queue_get(): : unsigned int hint, depth; : int nr; : : hint = this_cpu_read(*sbq->alloc_hint); 0.00 : ffff8000104b12c0: ldr x0, [x0, #24] : __read_once_size(): : __READ_ONCE_SIZE; 4.06 : ffff8000104b12c4: ldr w20, [x0, x2] 25.67 : ffff8000104b12c8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000104b12cc: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 10.18 : ffff8000104b12d0: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000104b12d4: cbz x0, ffff8000104b1324 <__sbitmap_queue_get+0x8c> : __read_once_size(): : __READ_ONCE_SIZE; 2.04 : ffff8000104b12d8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000104b12dc: cbz x0, ffff8000104b1324 <__sbitmap_queue_get+0x8c> : __read_once_size(): 30.16 : ffff8000104b12e0: ldr w21, [x19] : __sbitmap_queue_get(): : depth = READ_ONCE(sbq->sb.depth); : if (unlikely(hint >= depth)) { 0.00 : ffff8000104b12e4: cmp w20, w21 0.00 : ffff8000104b12e8: b.cs ffff8000104b1334 <__sbitmap_queue_get+0x9c> // b.hs, b.nlast : hint = depth ? prandom_u32() % depth : 0; : this_cpu_write(*sbq->alloc_hint, hint); : } : nr = sbitmap_get(&sbq->sb, hint, sbq->round_robin); 4.07 : ffff8000104b12ec: ldrb w2, [x19, #52] 0.00 : ffff8000104b12f0: mov w1, w20 0.00 : ffff8000104b12f4: mov x0, x19 0.00 : ffff8000104b12f8: bl ffff8000104b0848 : : if (nr == -1) { 0.00 : ffff8000104b12fc: cmn w0, #0x1 0.00 : ffff8000104b1300: b.eq ffff8000104b13d8 <__sbitmap_queue_get+0x140> // b.none : /* If the map is full, a hint won't do us much good. */ : this_cpu_write(*sbq->alloc_hint, 0); : } else if (nr == hint || unlikely(sbq->round_robin)) { 0.00 : ffff8000104b1304: cmp w0, w20 0.00 : ffff8000104b1308: b.eq ffff8000104b1378 <__sbitmap_queue_get+0xe0> // b.none 0.00 : ffff8000104b130c: ldrb w1, [x19, #52] 0.00 : ffff8000104b1310: cbnz w1, ffff8000104b1378 <__sbitmap_queue_get+0xe0> : hint = 0; : this_cpu_write(*sbq->alloc_hint, hint); : } : : return nr; : } 7.90 : ffff8000104b1314: ldp x19, x20, [sp, #16] 2.05 : ffff8000104b1318: ldr x21, [sp, #32] 0.00 : ffff8000104b131c: ldp x29, x30, [sp], #64 0.00 : ffff8000104b1320: ret : hint = this_cpu_read(*sbq->alloc_hint); 0.00 : ffff8000104b1324: bl ffff800010cd2e78 : __read_once_size(): 0.00 : ffff8000104b1328: ldr w21, [x19] : __sbitmap_queue_get(): : if (unlikely(hint >= depth)) { 0.00 : ffff8000104b132c: cmp w20, w21 0.00 : ffff8000104b1330: b.cc ffff8000104b12ec <__sbitmap_queue_get+0x54> // b.lo, b.ul, b.last : hint = depth ? prandom_u32() % depth : 0; 0.00 : ffff8000104b1334: mov w20, #0x0 // #0 0.00 : ffff8000104b1338: cbnz w21, ffff8000104b13f8 <__sbitmap_queue_get+0x160> : get_current(): 0.00 : ffff8000104b133c: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000104b1340: ldr w0, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000104b1344: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104b1348: str w0, [x1, #16] : __sbitmap_queue_get(): : this_cpu_write(*sbq->alloc_hint, hint); 0.00 : ffff8000104b134c: ldr x0, [x19, #24] : __my_cpu_offset(): 0.00 : ffff8000104b1350: mrs x2, tpidr_el1 : __write_once_size(): 0.00 : ffff8000104b1354: str w20, [x0, x2] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000104b1358: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000104b135c: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104b1360: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000104b1364: cbz x0, ffff8000104b1370 <__sbitmap_queue_get+0xd8> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000104b1368: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000104b136c: cbnz x0, ffff8000104b12ec <__sbitmap_queue_get+0x54> : __sbitmap_queue_get(): 0.00 : ffff8000104b1370: bl ffff800010cd2e78 0.00 : ffff8000104b1374: b ffff8000104b12ec <__sbitmap_queue_get+0x54> : hint = nr + 1; 0.00 : ffff8000104b1378: add w3, w0, #0x1 : if (hint >= depth - 1) 0.00 : ffff8000104b137c: sub w21, w21, #0x1 : get_current(): 0.00 : ffff8000104b1380: mrs x2, sp_el0 : __read_once_size(): 0.00 : ffff8000104b1384: ldr w1, [x2, #16] : __sbitmap_queue_get(): : hint = 0; 0.00 : ffff8000104b1388: cmp w3, w21 0.00 : ffff8000104b138c: csel w3, w3, wzr, cc // cc = lo, ul, last : __preempt_count_add(): : pc += val; 0.00 : ffff8000104b1390: add w1, w1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104b1394: str w1, [x2, #16] : __sbitmap_queue_get(): : this_cpu_write(*sbq->alloc_hint, hint); 0.00 : ffff8000104b1398: ldr x1, [x19, #24] : __my_cpu_offset(): 0.00 : ffff8000104b139c: mrs x4, tpidr_el1 : __write_once_size(): 0.00 : ffff8000104b13a0: str w3, [x1, x4] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000104b13a4: ldr x1, [x2, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000104b13a8: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104b13ac: str w1, [x2, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000104b13b0: cbz x1, ffff8000104b13bc <__sbitmap_queue_get+0x124> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000104b13b4: ldr x1, [x2, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000104b13b8: cbnz x1, ffff8000104b1314 <__sbitmap_queue_get+0x7c> 0.00 : ffff8000104b13bc: str w0, [x29, #60] : __sbitmap_queue_get(): : this_cpu_write(*sbq->alloc_hint, 0); 0.00 : ffff8000104b13c0: bl ffff800010cd2e78 0.00 : ffff8000104b13c4: ldr w0, [x29, #60] : } 0.00 : ffff8000104b13c8: ldp x19, x20, [sp, #16] 0.00 : ffff8000104b13cc: ldr x21, [sp, #32] 0.00 : ffff8000104b13d0: ldp x29, x30, [sp], #64 0.00 : ffff8000104b13d4: ret : get_current(): 0.00 : ffff8000104b13d8: mrs x2, sp_el0 : __read_once_size(): 0.00 : ffff8000104b13dc: ldr w1, [x2, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000104b13e0: add w1, w1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104b13e4: str w1, [x2, #16] : __sbitmap_queue_get(): : this_cpu_write(*sbq->alloc_hint, 0); 0.00 : ffff8000104b13e8: ldr x1, [x19, #24] : __my_cpu_offset(): 0.00 : ffff8000104b13ec: mrs x3, tpidr_el1 : __write_once_size(): 0.00 : ffff8000104b13f0: str wzr, [x1, x3] 0.00 : ffff8000104b13f4: b ffff8000104b13a4 <__sbitmap_queue_get+0x10c> : __sbitmap_queue_get(): : hint = depth ? prandom_u32() % depth : 0; 0.00 : ffff8000104b13f8: bl ffff800010484c08 0.00 : ffff8000104b13fc: udiv w20, w0, w21 0.00 : ffff8000104b1400: msub w20, w20, w21, w0 0.00 : ffff8000104b1404: b ffff8000104b133c <__sbitmap_queue_get+0xa4> Percent | Source code & Disassembly of vmlinux for cycles (21 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010127130 <_nohz_idle_balance>: : _nohz_idle_balance(): : * The function returns false if the loop has stopped before running : * through all idle CPUs. : */ : static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, : enum cpu_idle_type idle) : { 0.00 : ffff800010127130: stp x29, x30, [sp, #-144]! : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010127134: adrp x3, ffff800011afb000 : _nohz_idle_balance(): : /* Earliest time when we have to do rebalance again */ : unsigned long now = jiffies; : unsigned long next_balance = now + 60*HZ; 0.00 : ffff800010127138: mov x4, #0x3a98 // #15000 : { 0.00 : ffff80001012713c: mov x29, sp 0.00 : ffff800010127140: stp x19, x20, [sp, #16] : __write_once_size(): 0.00 : ffff800010127144: add x20, x3, #0xfc0 : _nohz_idle_balance(): : unsigned long now = jiffies; 0.00 : ffff800010127148: adrp x3, ffff800011907000 : { 0.00 : ffff80001012714c: stp x21, x22, [sp, #32] 0.00 : ffff800010127150: stp x23, x24, [sp, #48] : unsigned long now = jiffies; 0.00 : ffff800010127154: ldr x3, [x3, #2432] : { 0.00 : ffff800010127158: stp x25, x26, [sp, #64] 0.00 : ffff80001012715c: stp x27, x28, [sp, #80] : unsigned long next_balance = now + 60*HZ; 0.00 : ffff800010127160: add x19, x3, x4 : __write_once_size(): 0.00 : ffff800010127164: str wzr, [x20, #36] : _nohz_idle_balance(): : bool has_blocked_load = false; : int update_next_balance = 0; : int this_cpu = this_rq->cpu; 0.00 : ffff800010127168: ldr w23, [x0, #2568] : { 0.00 : ffff80001012716c: str x0, [x29, #104] 0.00 : ffff800010127170: str w2, [x29, #116] : unsigned long now = jiffies; 0.00 : ffff800010127174: str x3, [x29, #120] : { 0.00 : ffff800010127178: str w1, [x29, #136] : : /* : * Ensures that if we miss the CPU, we must see the has_blocked : * store from nohz_balance_enter_idle(). : */ : smp_mb(); 0.00 : ffff80001012717c: dmb ish : if (need_resched()) { : has_blocked_load = true; : goto abort; : } : : rq = cpu_rq(balance_cpu); 8.96 : ffff800010127180: adrp x25, ffff800011528000 0.00 : ffff800010127184: adrp x26, ffff800011909000 0.00 : ffff800010127188: add x25, x25, #0xe80 0.00 : ffff80001012718c: add x26, x26, #0x928 : for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { 0.00 : ffff800010127190: mov w27, #0xffffffff // #-1 : int update_next_balance = 0; 0.00 : ffff800010127194: mov w24, #0x0 // #0 : bool has_blocked_load = false; 0.00 : ffff800010127198: mov w21, #0x0 // #0 0.00 : ffff80001012719c: adrp x22, ffff80001190a000 <__per_cpu_offset+0x6d8> : for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { 0.00 : ffff8000101271a0: mov x1, x20 0.00 : ffff8000101271a4: mov w0, w27 0.00 : ffff8000101271a8: bl ffff800010cb92d8 9.27 : ffff8000101271ac: mov w27, w0 5.41 : ffff8000101271b0: ldr w1, [x22, #752] 0.00 : ffff8000101271b4: cmp w0, w1 0.00 : ffff8000101271b8: b.cs ffff80001012723c <_nohz_idle_balance+0x10c> // b.hs, b.nlast : if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) 5.04 : ffff8000101271bc: cmp w23, w27 0.00 : ffff8000101271c0: b.eq ffff8000101271a0 <_nohz_idle_balance+0x70> // b.none 14.47 : ffff8000101271c4: bl ffff800010118820 : rq = cpu_rq(balance_cpu); 20.38 : ffff8000101271c8: mov x3, x25 : : has_blocked_load |= update_nohz_stats(rq, true); 0.00 : ffff8000101271cc: mov w1, #0x1 // #1 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000101271d0: mrs x4, sp_el0 : _nohz_idle_balance(): : if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) 0.00 : ffff8000101271d4: cbz w0, ffff8000101271a0 <_nohz_idle_balance+0x70> : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000101271d8: ldr x0, [x4] : _nohz_idle_balance(): : if (need_resched()) { 0.00 : ffff8000101271dc: tbnz w0, #1, ffff8000101272cc <_nohz_idle_balance+0x19c> : rq = cpu_rq(balance_cpu); 9.32 : ffff8000101271e0: ldr x0, [x26, w27, sxtw #3] 0.00 : ffff8000101271e4: add x28, x3, x0 : has_blocked_load |= update_nohz_stats(rq, true); 0.00 : ffff8000101271e8: mov x0, x28 0.00 : ffff8000101271ec: bl ffff800010122dd0 : : /* : * If time for next balance is due, : * do the balance. : */ : if (time_after_eq(jiffies, rq->next_balance)) { 0.00 : ffff8000101271f0: adrp x1, ffff800011907000 : has_blocked_load |= update_nohz_stats(rq, true); 0.00 : ffff8000101271f4: and w0, w0, #0xff 4.63 : ffff8000101271f8: orr w21, w21, w0 : rq_lock_irqsave(): : : static inline void : rq_lock_irqsave(struct rq *rq, struct rq_flags *rf) : __acquires(rq->lock) : { : raw_spin_lock_irqsave(&rq->lock, rf->flags); 0.00 : ffff8000101271fc: mov x0, x28 : _nohz_idle_balance(): : if (time_after_eq(jiffies, rq->next_balance)) { 0.00 : ffff800010127200: ldr x4, [x1, #2432] 0.00 : ffff800010127204: ldr x1, [x28, #2376] 0.00 : ffff800010127208: sub x4, x4, x1 0.00 : ffff80001012720c: tbz x4, #63, ffff80001012729c <_nohz_idle_balance+0x16c> : if (flags & NOHZ_BALANCE_KICK) : rebalance_domains(rq, CPU_IDLE); : } : : if (time_after(next_balance, rq->next_balance)) { : next_balance = rq->next_balance; 11.84 : ffff800010127210: subs x0, x1, x19 0.00 : ffff800010127214: csel x19, x19, x1, pl // pl = nfrst 0.00 : ffff800010127218: cmp x0, #0x0 0.00 : ffff80001012721c: csinc w24, w24, wzr, ge // ge = tcont : for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { 0.00 : ffff800010127220: mov x1, x20 0.00 : ffff800010127224: mov w0, w27 0.00 : ffff800010127228: bl ffff800010cb92d8 0.00 : ffff80001012722c: mov w27, w0 10.69 : ffff800010127230: ldr w1, [x22, #752] 0.00 : ffff800010127234: cmp w0, w1 0.00 : ffff800010127238: b.cc ffff8000101271bc <_nohz_idle_balance+0x8c> // b.lo, b.ul, b.last : update_next_balance = 1; : } : } : : /* Newly idle CPU doesn't need an update */ : if (idle != CPU_NEWLY_IDLE) { 0.00 : ffff80001012723c: ldr w0, [x29, #116] 0.00 : ffff800010127240: cmp w0, #0x2 0.00 : ffff800010127244: b.ne ffff8000101272f8 <_nohz_idle_balance+0x1c8> // b.any : update_blocked_averages(this_cpu); : has_blocked_load |= this_rq->has_blocked_load; : } : : if (flags & NOHZ_BALANCE_KICK) 0.00 : ffff800010127248: ldr x0, [x29, #136] 0.00 : ffff80001012724c: tbnz w0, #0, ffff80001012731c <_nohz_idle_balance+0x1ec> : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010127250: adrp x0, ffff800011afb000 0.00 : ffff800010127254: add x1, x0, #0xfc0 : _nohz_idle_balance(): : rebalance_domains(this_rq, CPU_IDLE); : : WRITE_ONCE(nohz.next_blocked, 0.00 : ffff800010127258: ldr x0, [x29, #120] 0.00 : ffff80001012725c: add x0, x0, #0x8 : __write_once_size(): 0.00 : ffff800010127260: str x0, [x1, #48] : _nohz_idle_balance(): : /* The full idle balance loop has been done */ : ret = true; : : abort: : /* There is still blocked load, enable periodic update */ : if (has_blocked_load) 0.00 : ffff800010127264: cbnz w21, ffff8000101272d0 <_nohz_idle_balance+0x1a0> 0.00 : ffff800010127268: mov w21, #0x1 // #1 : /* : * next_balance will be updated only when there is a need. : * When the CPU is attached to null domain for ex, it will not be : * updated. : */ : if (likely(update_next_balance)) 0.00 : ffff80001012726c: cbz w24, ffff80001012727c <_nohz_idle_balance+0x14c> : nohz.next_balance = next_balance; 0.00 : ffff800010127270: adrp x0, ffff800011afb000 0.00 : ffff800010127274: add x27, x0, #0xfc0 0.00 : ffff800010127278: str x19, [x27, #40] : : return ret; : } 0.00 : ffff80001012727c: mov w0, w21 0.00 : ffff800010127280: ldp x19, x20, [sp, #16] 0.00 : ffff800010127284: ldp x21, x22, [sp, #32] 0.00 : ffff800010127288: ldp x23, x24, [sp, #48] 0.00 : ffff80001012728c: ldp x25, x26, [sp, #64] 0.00 : ffff800010127290: ldp x27, x28, [sp, #80] 0.00 : ffff800010127294: ldp x29, x30, [sp], #144 0.00 : ffff800010127298: ret : rq_lock_irqsave(): 0.00 : ffff80001012729c: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff8000101272a0: mov x1, x0 : _nohz_idle_balance(): : update_rq_clock(rq); 0.00 : ffff8000101272a4: mov x0, x28 : rq_lock_irqsave(): 0.00 : ffff8000101272a8: str x1, [x29, #128] : _nohz_idle_balance(): 0.00 : ffff8000101272ac: bl ffff800010114980 : rq_unlock_irqrestore(): : static inline void : rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf) : __releases(rq->lock) : { : rq_unpin_lock(rq, rf); : raw_spin_unlock_irqrestore(&rq->lock, rf->flags); 0.00 : ffff8000101272b0: ldr x1, [x29, #128] 0.00 : ffff8000101272b4: mov x0, x28 0.00 : ffff8000101272b8: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : _nohz_idle_balance(): : if (flags & NOHZ_BALANCE_KICK) 0.00 : ffff8000101272bc: ldr x0, [x29, #136] 0.00 : ffff8000101272c0: tbnz w0, #0, ffff8000101272e4 <_nohz_idle_balance+0x1b4> 0.00 : ffff8000101272c4: ldr x1, [x28, #2376] 0.00 : ffff8000101272c8: b ffff800010127210 <_nohz_idle_balance+0xe0> 0.00 : ffff8000101272cc: mov w21, #0x0 // #0 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101272d0: adrp x0, ffff800011afb000 0.00 : ffff8000101272d4: add x0, x0, #0xfc0 0.00 : ffff8000101272d8: mov w1, #0x1 // #1 0.00 : ffff8000101272dc: str w1, [x0, #36] 0.00 : ffff8000101272e0: b ffff80001012726c <_nohz_idle_balance+0x13c> : _nohz_idle_balance(): : rebalance_domains(rq, CPU_IDLE); 0.00 : ffff8000101272e4: mov w1, #0x0 // #0 0.00 : ffff8000101272e8: mov x0, x28 0.00 : ffff8000101272ec: bl ffff800010126d70 0.00 : ffff8000101272f0: ldr x1, [x28, #2376] 0.00 : ffff8000101272f4: b ffff800010127210 <_nohz_idle_balance+0xe0> : update_blocked_averages(this_cpu); 0.00 : ffff8000101272f8: mov w0, w23 0.00 : ffff8000101272fc: bl ffff8000101226f8 : has_blocked_load |= this_rq->has_blocked_load; 0.00 : ffff800010127300: ldr x0, [x29, #104] 0.00 : ffff800010127304: ldr w0, [x0, #40] 0.00 : ffff800010127308: orr w21, w21, w0 : if (flags & NOHZ_BALANCE_KICK) 0.00 : ffff80001012730c: ldr x0, [x29, #136] : has_blocked_load |= this_rq->has_blocked_load; 0.00 : ffff800010127310: cmp w21, #0x0 0.00 : ffff800010127314: cset w21, ne // ne = any : if (flags & NOHZ_BALANCE_KICK) 0.00 : ffff800010127318: tbz w0, #0, ffff800010127250 <_nohz_idle_balance+0x120> : rebalance_domains(this_rq, CPU_IDLE); 0.00 : ffff80001012731c: ldr x0, [x29, #104] 0.00 : ffff800010127320: mov w1, #0x0 // #0 0.00 : ffff800010127324: bl ffff800010126d70 0.00 : ffff800010127328: b ffff800010127250 <_nohz_idle_balance+0x120> Percent | Source code & Disassembly of vmlinux for cycles (51 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001045cc00 : : blk_flush_plug_list(): : return cb; : } : EXPORT_SYMBOL(blk_check_plugged); : : void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) : { 7.88 : ffff80001045cc00: stp x29, x30, [sp, #-112]! 0.00 : ffff80001045cc04: mov x29, sp 21.68 : ffff80001045cc08: stp x19, x20, [sp, #16] 0.00 : ffff80001045cc0c: and w20, w1, #0xff 3.91 : ffff80001045cc10: stp x21, x22, [sp, #32] : list_del(): : * in an undefined state. : */ : static inline void list_del(struct list_head *entry) : { : __list_del_entry(entry); : entry->next = LIST_POISON1; 0.00 : ffff80001045cc14: mov x22, #0x100 // #256 : blk_flush_plug_list(): 0.00 : ffff80001045cc18: stp x23, x24, [sp, #48] 0.00 : ffff80001045cc1c: mov x23, x0 3.90 : ffff80001045cc20: str x25, [sp, #64] 0.00 : ffff80001045cc24: add x24, x0, #0x10 0.00 : ffff80001045cc28: adrp x25, ffff800011909000 0.00 : ffff80001045cc2c: add x0, x25, #0x908 : list_del(): : entry->prev = LIST_POISON2; 3.95 : ffff80001045cc30: mov x21, #0x122 // #290 : blk_flush_plug_list(): 0.00 : ffff80001045cc34: ldr x1, [x0] 3.92 : ffff80001045cc38: str x1, [x29, #104] 0.00 : ffff80001045cc3c: mov x1, #0x0 // #0 : flush_plug_callbacks(): : LIST_HEAD(callbacks); 1.93 : ffff80001045cc40: add x19, x29, #0x58 : list_del(): : entry->next = LIST_POISON1; 0.00 : ffff80001045cc44: movk x22, #0xdead, lsl #48 : entry->prev = LIST_POISON2; 0.00 : ffff80001045cc48: movk x21, #0xdead, lsl #48 : flush_plug_callbacks(): 0.00 : ffff80001045cc4c: stp x19, x19, [x29, #88] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 33.27 : ffff80001045cc50: ldr x0, [x23, #16] : flush_plug_callbacks(): : while (!list_empty(&plug->cb_list)) { 0.00 : ffff80001045cc54: cmp x24, x0 0.00 : ffff80001045cc58: b.eq ffff80001045ccbc // b.none : __read_once_size(): 0.00 : ffff80001045cc5c: ldr x0, [x23, #16] : list_splice_init(): : * The list at @list is reinitialised : */ : static inline void list_splice_init(struct list_head *list, : struct list_head *head) : { : if (!list_empty(list)) { 0.00 : ffff80001045cc60: cmp x24, x0 0.00 : ffff80001045cc64: b.eq ffff80001045cc88 // b.none : __list_splice(): : struct list_head *last = list->prev; 0.00 : ffff80001045cc68: ldp x2, x1, [x23, #16] : first->prev = prev; 0.00 : ffff80001045cc6c: str x19, [x2, #8] : list_splice_init(): : __list_splice(list, head, head->next); 0.00 : ffff80001045cc70: ldr x0, [x29, #88] : __list_splice(): : prev->next = first; 0.00 : ffff80001045cc74: str x2, [x29, #88] : last->next = next; 0.00 : ffff80001045cc78: str x0, [x1] : next->prev = last; 0.00 : ffff80001045cc7c: str x1, [x0, #8] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001045cc80: str x24, [x23, #16] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff80001045cc84: str x24, [x23, #24] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001045cc88: ldr x0, [x29, #88] : flush_plug_callbacks(): : while (!list_empty(&callbacks)) { 0.00 : ffff80001045cc8c: cmp x19, x0 0.00 : ffff80001045cc90: b.eq ffff80001045cc50 // b.none : struct blk_plug_cb *cb = list_first_entry(&callbacks, 0.00 : ffff80001045cc94: ldr x2, [x29, #88] : cb->callback(cb, from_schedule); 0.00 : ffff80001045cc98: mov w1, w20 0.00 : ffff80001045cc9c: mov x0, x2 : __list_del_entry(): : __list_del(entry->prev, entry->next); 0.00 : ffff80001045cca0: ldp x4, x3, [x2] : __list_del(): : next->prev = prev; 0.00 : ffff80001045cca4: str x3, [x4, #8] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001045cca8: str x4, [x3] : list_del(): : entry->prev = LIST_POISON2; 0.00 : ffff80001045ccac: stp x22, x21, [x2] : flush_plug_callbacks(): 0.00 : ffff80001045ccb0: ldr x2, [x2, #16] 0.00 : ffff80001045ccb4: blr x2 0.00 : ffff80001045ccb8: b ffff80001045cc88 : __read_once_size(): : __READ_ONCE_SIZE; 7.80 : ffff80001045ccbc: ldr x0, [x23] : blk_flush_plug_list(): : flush_plug_callbacks(plug, from_schedule); : : if (!list_empty(&plug->mq_list)) 5.85 : ffff80001045ccc0: cmp x23, x0 0.00 : ffff80001045ccc4: b.eq ffff80001045ccd4 // b.none : blk_mq_flush_plug_list(plug, from_schedule); 3.94 : ffff80001045ccc8: mov w1, w20 0.00 : ffff80001045cccc: mov x0, x23 0.00 : ffff80001045ccd0: bl ffff800010467df0 : } 0.00 : ffff80001045ccd4: add x25, x25, #0x908 0.00 : ffff80001045ccd8: ldr x1, [x29, #104] 0.00 : ffff80001045ccdc: ldr x0, [x25] 0.00 : ffff80001045cce0: eor x0, x1, x0 0.00 : ffff80001045cce4: cbnz x0, ffff80001045cd00 1.96 : ffff80001045cce8: ldp x19, x20, [sp, #16] 0.00 : ffff80001045ccec: ldp x21, x22, [sp, #32] 0.00 : ffff80001045ccf0: ldp x23, x24, [sp, #48] 0.00 : ffff80001045ccf4: ldr x25, [sp, #64] 0.00 : ffff80001045ccf8: ldp x29, x30, [sp], #112 0.00 : ffff80001045ccfc: ret 0.00 : ffff80001045cd00: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (58 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010456648 <__bio_associate_blkg.isra.40>: : __bio_associate_blkg(): : * alive blkg. : * : * A reference will be taken on the @blkg and will be released when @bio is : * freed. : */ : static void __bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg) 3.45 : ffff800010456648: stp x29, x30, [sp, #-48]! 0.00 : ffff80001045664c: mov x29, sp 1.73 : ffff800010456650: stp x19, x20, [sp, #16] 0.00 : ffff800010456654: mov x20, x0 0.00 : ffff800010456658: str x21, [sp, #32] 0.00 : ffff80001045665c: mov x19, x1 : bio_disassociate_blkg(): : if (bio->bi_blkg) { 1.75 : ffff800010456660: ldr x21, [x0] 0.00 : ffff800010456664: cbz x21, ffff8000104566c0 <__bio_associate_blkg.isra.40+0x78> : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff800010456668: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001045666c: ldr x0, [x21, #72] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff800010456670: tst x0, #0x3 0.00 : ffff800010456674: b.ne ffff8000104567b0 <__bio_associate_blkg.isra.40+0x168> // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010456678: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff80001045667c: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010456680: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010456684: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff800010456688: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001045668c: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff800010456690: add x0, x0, x2 0.00 : ffff800010456694: ldxr x5, [x0] 0.00 : ffff800010456698: add x5, x5, x3 0.00 : ffff80001045669c: stxr w4, x5, [x0] 0.00 : ffff8000104566a0: cbnz w4, ffff800010456694 <__bio_associate_blkg.isra.40+0x4c> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000104566a4: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000104566a8: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104566ac: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000104566b0: cbnz x0, ffff8000104567a4 <__bio_associate_blkg.isra.40+0x15c> : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff8000104566b4: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff8000104566b8: bl ffff800010160d20 <__rcu_read_unlock> : bio_disassociate_blkg(): : bio->bi_blkg = NULL; 0.00 : ffff8000104566bc: str xzr, [x20] : blkg_tryget_closest(): : { : struct blkcg_gq *ret_blkg = NULL; : : WARN_ON_ONCE(!rcu_read_lock_held()); : : while (blkg) { 3.23 : ffff8000104566c0: cbz x19, ffff80001045678c <__bio_associate_blkg.isra.40+0x144> 1.78 : ffff8000104566c4: nop : rcu_read_lock(): : __rcu_read_lock(); 0.00 : ffff8000104566c8: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000104566cc: ldr x0, [x19, #72] : __ref_is_percpu(): : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000104566d0: tst x0, #0x3 0.00 : ffff8000104566d4: b.ne ffff80001045673c <__bio_associate_blkg.isra.40+0xf4> // b.any : get_current(): 0.00 : ffff8000104566d8: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000104566dc: ldr w2, [x1, #16] : __preempt_count_add(): : pc += val; 1.73 : ffff8000104566e0: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 1.75 : ffff8000104566e4: str w2, [x1, #16] : __percpu_add_case_64(): 0.00 : ffff8000104566e8: mov x3, #0x1 // #1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000104566ec: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff8000104566f0: add x0, x0, x2 1.73 : ffff8000104566f4: ldxr x5, [x0] 38.04 : ffff8000104566f8: add x5, x5, x3 0.00 : ffff8000104566fc: stxr w4, x5, [x0] 0.00 : ffff800010456700: cbnz w4, ffff8000104566f4 <__bio_associate_blkg.isra.40+0xac> : __read_once_size(): : __READ_ONCE_SIZE; 13.69 : ffff800010456704: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010456708: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 12.20 : ffff80001045670c: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010456710: cbz x0, ffff800010456734 <__bio_associate_blkg.isra.40+0xec> : __read_once_size(): : __READ_ONCE_SIZE; 3.47 : ffff800010456714: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010456718: cbz x0, ffff800010456734 <__bio_associate_blkg.isra.40+0xec> : rcu_read_unlock(): : __rcu_read_unlock(); 8.64 : ffff80001045671c: bl ffff800010160d20 <__rcu_read_unlock> : __bio_associate_blkg(): : { : bio_disassociate_blkg(bio); : : bio->bi_blkg = blkg_tryget_closest(blkg); 5.19 : ffff800010456720: str x19, [x20] : } 1.61 : ffff800010456724: ldp x19, x20, [sp, #16] 0.00 : ffff800010456728: ldr x21, [sp, #32] 0.00 : ffff80001045672c: ldp x29, x30, [sp], #48 0.00 : ffff800010456730: ret : percpu_ref_tryget_many(): : this_cpu_add(*percpu_count, nr); 0.00 : ffff800010456734: bl ffff800010cd2e78 0.00 : ffff800010456738: b ffff80001045671c <__bio_associate_blkg.isra.40+0xd4> : __read_once_size(): 0.00 : ffff80001045673c: ldr x3, [x19, #64] : atomic64_fetch_add_unless(): : atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) : { : s64 c = atomic64_read(v); : : do { : if (unlikely(c == u)) 0.00 : ffff800010456740: cbz x3, ffff800010456780 <__bio_associate_blkg.isra.40+0x138> 0.00 : ffff800010456744: add x4, x19, #0x40 : break; : } while (!atomic64_try_cmpxchg(v, &c, c + a)); 0.00 : ffff800010456748: add x2, x3, #0x1 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001045674c: b ffff80001045676c <__bio_associate_blkg.isra.40+0x124> 0.00 : ffff800010456750: b ffff80001045676c <__bio_associate_blkg.isra.40+0x124> : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff800010456754: mov x0, x4 0.00 : ffff800010456758: mov x1, x3 0.00 : ffff80001045675c: mov x5, x1 0.00 : ffff800010456760: casal x5, x2, [x4] 0.00 : ffff800010456764: mov x0, x5 0.00 : ffff800010456768: b ffff800010456770 <__bio_associate_blkg.isra.40+0x128> : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff80001045676c: b ffff800010458b40 : atomic64_try_cmpxchg(): : if (unlikely(r != o)) 0.00 : ffff800010456770: cmp x0, x3 0.00 : ffff800010456774: b.eq ffff80001045671c <__bio_associate_blkg.isra.40+0xd4> // b.none 0.00 : ffff800010456778: mov x3, x0 : atomic64_fetch_add_unless(): : if (unlikely(c == u)) 0.00 : ffff80001045677c: cbnz x0, ffff800010456748 <__bio_associate_blkg.isra.40+0x100> : rcu_read_unlock(): 0.00 : ffff800010456780: bl ffff800010160d20 <__rcu_read_unlock> : blkg_tryget_closest(): : if (blkg_tryget(blkg)) { : ret_blkg = blkg; : break; : } : blkg = blkg->parent; 0.00 : ffff800010456784: ldr x19, [x19, #56] : while (blkg) { 0.00 : ffff800010456788: cbnz x19, ffff8000104566c8 <__bio_associate_blkg.isra.40+0x80> : struct blkcg_gq *ret_blkg = NULL; 0.00 : ffff80001045678c: mov x19, #0x0 // #0 : __bio_associate_blkg(): : bio->bi_blkg = blkg_tryget_closest(blkg); 0.00 : ffff800010456790: str x19, [x20] : } 0.00 : ffff800010456794: ldp x19, x20, [sp, #16] 0.00 : ffff800010456798: ldr x21, [sp, #32] 0.00 : ffff80001045679c: ldp x29, x30, [sp], #48 0.00 : ffff8000104567a0: ret : __read_once_size(): 0.00 : ffff8000104567a4: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000104567a8: cbz x0, ffff8000104566b4 <__bio_associate_blkg.isra.40+0x6c> 0.00 : ffff8000104567ac: b ffff8000104566b8 <__bio_associate_blkg.isra.40+0x70> : blkg_put(): : * blkg_put - put a blkg reference : * @blkg: blkg to put : */ : static inline void blkg_put(struct blkcg_gq *blkg) : { : percpu_ref_put(&blkg->refcnt); 0.00 : ffff8000104567b0: add x0, x21, #0x40 : arch_static_branch_jump(): 0.00 : ffff8000104567b4: b ffff8000104567dc <__bio_associate_blkg.isra.40+0x194> 0.00 : ffff8000104567b8: b ffff8000104567dc <__bio_associate_blkg.isra.40+0x194> : __lse_atomic64_sub_return(): : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000104567bc: mov x1, #0x1 // #1 0.00 : ffff8000104567c0: neg x1, x1 0.00 : ffff8000104567c4: ldaddal x1, x2, [x0] 0.00 : ffff8000104567c8: add x1, x1, x2 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000104567cc: cbnz x1, ffff8000104566b8 <__bio_associate_blkg.isra.40+0x70> : ref->release(ref); 0.00 : ffff8000104567d0: ldr x1, [x0, #16] 0.00 : ffff8000104567d4: blr x1 0.00 : ffff8000104567d8: b ffff8000104566b8 <__bio_associate_blkg.isra.40+0x70> : __ll_sc_atomic64_sub_return(): : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff8000104567dc: mov x2, #0x1 // #1 0.00 : ffff8000104567e0: add x4, x21, #0x40 0.00 : ffff8000104567e4: b ffff800010458b60 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000104567e8: cbnz x1, ffff8000104566b8 <__bio_associate_blkg.isra.40+0x70> 0.00 : ffff8000104567ec: b ffff8000104567d0 <__bio_associate_blkg.isra.40+0x188> Percent | Source code & Disassembly of vmlinux for cycles (52 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001046a0c0 <__blk_mq_get_tag>: : __blk_mq_get_tag(): : return atomic_read(&hctx->nr_active) < depth; : } : : static int __blk_mq_get_tag(struct blk_mq_alloc_data *data, : struct sbitmap_queue *bt) : { 96.10 : ffff80001046a0c0: mov x2, x0 : if (!(data->flags & BLK_MQ_REQ_INTERNAL) && 0.00 : ffff80001046a0c4: ldr w0, [x0, #8] 0.00 : ffff80001046a0c8: tbz w0, #2, ffff80001046a0f8 <__blk_mq_get_tag+0x38> : { 1.94 : ffff80001046a0cc: stp x29, x30, [sp, #-16]! 0.00 : ffff80001046a0d0: mov x0, x1 0.00 : ffff80001046a0d4: mov x29, sp : !hctx_may_queue(data->hctx, bt)) : return -1; : if (data->shallow_depth) 0.00 : ffff80001046a0d8: ldr w1, [x2, #12] 0.00 : ffff80001046a0dc: cbnz w1, ffff80001046a0ec <__blk_mq_get_tag+0x2c> : return __sbitmap_queue_get_shallow(bt, data->shallow_depth); : else : return __sbitmap_queue_get(bt); 0.00 : ffff80001046a0e0: bl ffff8000104b1298 <__sbitmap_queue_get> : } 1.95 : ffff80001046a0e4: ldp x29, x30, [sp], #16 0.00 : ffff80001046a0e8: ret : return __sbitmap_queue_get_shallow(bt, data->shallow_depth); 0.00 : ffff80001046a0ec: bl ffff8000104b1110 <__sbitmap_queue_get_shallow> : } 0.00 : ffff80001046a0f0: ldp x29, x30, [sp], #16 0.00 : ffff80001046a0f4: ret : !hctx_may_queue(data->hctx, bt)) 0.00 : ffff80001046a0f8: ldr x0, [x2, #32] : hctx_may_queue(): : if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED)) 0.00 : ffff80001046a0fc: cbz x0, ffff80001046a0cc <__blk_mq_get_tag+0xc> 0.00 : ffff80001046a100: ldr x3, [x0, #192] 0.00 : ffff80001046a104: tbz w3, #1, ffff80001046a0cc <__blk_mq_get_tag+0xc> : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001046a108: ldr x3, [x0, #24] : hctx_may_queue(): : if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) 0.00 : ffff80001046a10c: tst w3, #0x2 0.00 : ffff80001046a110: b.eq ffff80001046a0cc <__blk_mq_get_tag+0xc> // b.none : if (bt->sb.depth == 1) 0.00 : ffff80001046a114: ldr w3, [x1] 0.00 : ffff80001046a118: cmp w3, #0x1 0.00 : ffff80001046a11c: b.eq ffff80001046a0cc <__blk_mq_get_tag+0xc> // b.none : users = atomic_read(&hctx->tags->active_queues); 0.00 : ffff80001046a120: ldr x4, [x0, #336] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001046a124: ldr w4, [x4, #8] : hctx_may_queue(): : if (!users) 0.00 : ffff80001046a128: cbz w4, ffff80001046a0cc <__blk_mq_get_tag+0xc> : depth = max((bt->sb.depth + users - 1) / users, 4U); 0.00 : ffff80001046a12c: sub w3, w3, #0x1 : __read_once_size(): 0.00 : ffff80001046a130: ldr w5, [x0, #432] : hctx_may_queue(): 0.00 : ffff80001046a134: add w3, w3, w4 0.00 : ffff80001046a138: mov w6, #0x4 // #4 : __blk_mq_get_tag(): : return -1; 0.00 : ffff80001046a13c: mov w0, #0xffffffff // #-1 : hctx_may_queue(): : depth = max((bt->sb.depth + users - 1) / users, 4U); 0.00 : ffff80001046a140: udiv w3, w3, w4 0.00 : ffff80001046a144: cmp w3, w6 0.00 : ffff80001046a148: csel w3, w3, w6, cs // cs = hs, nlast : __blk_mq_get_tag(): : if (!(data->flags & BLK_MQ_REQ_INTERNAL) && 0.00 : ffff80001046a14c: cmp w3, w5 0.00 : ffff80001046a150: b.hi ffff80001046a0cc <__blk_mq_get_tag+0xc> // b.pmore : } 0.00 : ffff80001046a154: ret Percent | Source code & Disassembly of vmlinux for cycles (53 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001008725c : : fpsimd_save_state(): : * Save the FP registers. : * : * x0 - pointer to struct fpsimd_state : */ : ENTRY(fpsimd_save_state) : fpsimd_save x0, 8 0.00 : ffff80001008725c: stp q0, q1, [x0] 3.79 : ffff800010087260: stp q2, q3, [x0, #32] 5.72 : ffff800010087264: stp q4, q5, [x0, #64] 1.72 : ffff800010087268: stp q6, q7, [x0, #96] 0.00 : ffff80001008726c: stp q8, q9, [x0, #128] 1.88 : ffff800010087270: stp q10, q11, [x0, #160] 0.00 : ffff800010087274: stp q12, q13, [x0, #192] 1.94 : ffff800010087278: stp q14, q15, [x0, #224] 0.00 : ffff80001008727c: stp q16, q17, [x0, #256] 1.91 : ffff800010087280: stp q18, q19, [x0, #288] 3.77 : ffff800010087284: stp q20, q21, [x0, #320] 1.68 : ffff800010087288: stp q22, q23, [x0, #352] 15.14 : ffff80001008728c: stp q24, q25, [x0, #384] 35.88 : ffff800010087290: stp q26, q27, [x0, #416] 3.78 : ffff800010087294: stp q28, q29, [x0, #448] 15.19 : ffff800010087298: stp q30, q31, [x0, #480]! 0.00 : ffff80001008729c: mrs x8, fpsr 0.00 : ffff8000100872a0: str w8, [x0, #32] 7.59 : ffff8000100872a4: mrs x8, fpcr 0.00 : ffff8000100872a8: str w8, [x0, #36] : ret 0.00 : ffff8000100872ac: ret Percent | Source code & Disassembly of vmlinux for cycles (63 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010714300 : : iommu_dma_alloc_iova(): : } : : static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, : size_t size, u64 dma_limit, struct device *dev) : { : struct iommu_dma_cookie *cookie = domain->iova_cookie; 0.00 : ffff800010714300: ldr x5, [x0, #64] : struct iova_domain *iovad = &cookie->iovad; : unsigned long shift, iova_len, iova = 0; : : if (cookie->type == IOMMU_DMA_MSI_COOKIE) { 12.65 : ffff800010714304: ldr w6, [x5] 0.00 : ffff800010714308: cmp w6, #0x1 0.00 : ffff80001071430c: b.eq ffff8000107143d0 // b.none : static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, 15.92 : ffff800010714310: stp x29, x30, [sp, #-48]! 0.00 : ffff800010714314: mov x29, sp 1.61 : ffff800010714318: stp x19, x20, [sp, #16] 0.00 : ffff80001071431c: stp x21, x22, [sp, #32] : __ffs(): : * : * Undefined if no bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __ffs(unsigned long word) : { : return __builtin_ctzl(word); 0.00 : ffff800010714320: ldr x21, [x5, #40] 0.00 : ffff800010714324: rbit x21, x21 0.00 : ffff800010714328: clz x21, x21 : iommu_dma_alloc_iova(): : cookie->msi_iova += size; : return cookie->msi_iova - size; : } : : shift = iova_shift(iovad); : iova_len = size >> shift; 0.00 : ffff80001071432c: lsr x19, x1, x21 : * Freeing non-power-of-two-sized allocations back into the IOVA caches : * will come back to bite us badly, so we have to waste a bit of space : * rounding up anything cacheable to make sure that can't happen. The : * order of the unadjusted size will still match upon freeing. : */ : if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) 1.55 : ffff800010714330: cmp x19, #0x1f 0.00 : ffff800010714334: b.hi ffff80001071435c // b.pmore : __roundup_pow_of_two(): : * @n: value to round up : */ : static inline __attribute__((const)) : unsigned long __roundup_pow_of_two(unsigned long n) : { : return 1UL << fls_long(n - 1); 1.60 : ffff800010714338: sub x6, x19, #0x1 0.00 : ffff80001071433c: mov x1, #0x3f // #63 1.58 : ffff800010714340: clz x7, x6 : fls64(): : return fls(x); : } : #elif BITS_PER_LONG == 64 : static __always_inline int fls64(__u64 x) : { : if (x == 0) 0.00 : ffff800010714344: mov x19, #0x1 // #1 0.00 : ffff800010714348: sub x1, x1, x7 0.00 : ffff80001071434c: cmp x6, #0x0 0.00 : ffff800010714350: add w1, w1, #0x1 0.00 : ffff800010714354: lsl x1, x19, x1 0.00 : ffff800010714358: csel x19, x1, x19, ne // ne = any : iommu_dma_alloc_iova(): : iova_len = roundup_pow_of_two(iova_len); : : dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); 0.00 : ffff80001071435c: ldr x22, [x4] 1.62 : ffff800010714360: cbz x2, ffff800010714370 4.82 : ffff800010714364: cbz x22, ffff8000107143e0 0.00 : ffff800010714368: cmp x22, x2 0.00 : ffff80001071436c: csel x22, x22, x2, ls // ls = plast : : if (domain->geometry.force_aperture) 0.00 : ffff800010714370: ldrb w1, [x0, #56] 0.00 : ffff800010714374: cbz w1, ffff800010714384 : dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end); 0.00 : ffff800010714378: ldr x0, [x0, #48] 0.00 : ffff80001071437c: cmp x22, x0 6.37 : ffff800010714380: csel x22, x22, x0, ls // ls = plast : : /* Try to get PCI devices a SAC address */ : if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev)) 0.00 : ffff800010714384: mov x2, #0xffffffff // #4294967295 : struct iova_domain *iovad = &cookie->iovad; 4.79 : ffff800010714388: add x20, x5, #0x8 : if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev)) 0.00 : ffff80001071438c: cmp x22, x2 3.22 : ffff800010714390: b.ls ffff8000107143a8 // b.plast 0.00 : ffff800010714394: ldr x1, [x3] 0.00 : ffff800010714398: adrp x0, ffff80001199d000 0.00 : ffff80001071439c: add x0, x0, #0x740 0.00 : ffff8000107143a0: cmp x1, x0 18.99 : ffff8000107143a4: b.eq ffff8000107143f0 // b.none : iova = alloc_iova_fast(iovad, iova_len, : DMA_BIT_MASK(32) >> shift, false); : : if (!iova) : iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, 0.00 : ffff8000107143a8: mov w3, #0x1 // #1 0.00 : ffff8000107143ac: lsr x2, x22, x21 0.00 : ffff8000107143b0: mov x1, x19 0.00 : ffff8000107143b4: mov x0, x20 0.00 : ffff8000107143b8: bl ffff800010718568 : true); : : return (dma_addr_t)iova << shift; 3.07 : ffff8000107143bc: lsl x0, x0, x21 : } 0.00 : ffff8000107143c0: ldp x19, x20, [sp, #16] 1.60 : ffff8000107143c4: ldp x21, x22, [sp, #32] 0.00 : ffff8000107143c8: ldp x29, x30, [sp], #48 0.00 : ffff8000107143cc: ret : cookie->msi_iova += size; 0.00 : ffff8000107143d0: ldr x0, [x5, #8] 0.00 : ffff8000107143d4: add x1, x0, x1 0.00 : ffff8000107143d8: str x1, [x5, #8] : } 0.00 : ffff8000107143dc: ret : if (domain->geometry.force_aperture) 4.77 : ffff8000107143e0: ldrb w1, [x0, #56] : dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); 0.00 : ffff8000107143e4: mov x22, x2 : if (domain->geometry.force_aperture) 0.00 : ffff8000107143e8: cbz w1, ffff800010714384 14.35 : ffff8000107143ec: b ffff800010714378 : iova = alloc_iova_fast(iovad, iova_len, 0.00 : ffff8000107143f0: mov w3, #0x0 // #0 0.00 : ffff8000107143f4: lsr x2, x2, x21 0.00 : ffff8000107143f8: mov x1, x19 0.00 : ffff8000107143fc: mov x0, x20 1.49 : ffff800010714400: bl ffff800010718568 : if (!iova) 0.00 : ffff800010714404: cbz x0, ffff8000107143a8 0.00 : ffff800010714408: b ffff8000107143bc Percent | Source code & Disassembly of vmlinux for cycles (15 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011cfb0 : : clear_buddies(): : } : } : : static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) : { : if (cfs_rq->last == se) 13.32 : ffff80001011cfb0: ldr x2, [x0, #88] 0.00 : ffff80001011cfb4: cmp x2, x1 0.00 : ffff80001011cfb8: b.eq ffff80001011d04c // b.none : __clear_buddies_last(se); : : if (cfs_rq->next == se) 39.91 : ffff80001011cfbc: ldr x2, [x0, #80] 6.66 : ffff80001011cfc0: cmp x2, x1 0.00 : ffff80001011cfc4: b.eq ffff80001011d010 // b.none : __clear_buddies_next(se); : : if (cfs_rq->skip == se) 19.92 : ffff80001011cfc8: ldr x0, [x0, #96] 0.00 : ffff80001011cfcc: cmp x0, x1 0.00 : ffff80001011cfd0: b.eq ffff80001011cfd8 // b.none : __clear_buddies_skip(se); : } 20.19 : ffff80001011cfd4: ret : __clear_buddies_skip(): : for_each_sched_entity(se) { 0.00 : ffff80001011cfd8: cbz x1, ffff80001011cfd4 : struct cfs_rq *cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011cfdc: ldr x0, [x1, #128] : if (cfs_rq->skip != se) 0.00 : ffff80001011cfe0: ldr x2, [x0, #96] 0.00 : ffff80001011cfe4: cmp x1, x2 0.00 : ffff80001011cfe8: b.eq ffff80001011d000 // b.none 0.00 : ffff80001011cfec: b ffff80001011cfd4 : struct cfs_rq *cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011cff0: ldr x0, [x1, #128] : if (cfs_rq->skip != se) 0.00 : ffff80001011cff4: ldr x2, [x0, #96] 0.00 : ffff80001011cff8: cmp x2, x1 0.00 : ffff80001011cffc: b.ne ffff80001011cfd4 // b.any : cfs_rq->skip = NULL; 0.00 : ffff80001011d000: str xzr, [x0, #96] : for_each_sched_entity(se) { 0.00 : ffff80001011d004: ldr x1, [x1, #120] 0.00 : ffff80001011d008: cbnz x1, ffff80001011cff0 : clear_buddies(): : } 0.00 : ffff80001011d00c: ret : __clear_buddies_next(): : for_each_sched_entity(se) { 0.00 : ffff80001011d010: cbz x1, ffff80001011cfd4 : struct cfs_rq *cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011d014: ldr x3, [x1, #128] : if (cfs_rq->next != se) 0.00 : ffff80001011d018: ldr x2, [x3, #80] 0.00 : ffff80001011d01c: cmp x1, x2 0.00 : ffff80001011d020: b.ne ffff80001011cfc8 // b.any 0.00 : ffff80001011d024: mov x2, x1 0.00 : ffff80001011d028: b ffff80001011d03c : struct cfs_rq *cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011d02c: ldr x3, [x2, #128] : if (cfs_rq->next != se) 0.00 : ffff80001011d030: ldr x4, [x3, #80] 0.00 : ffff80001011d034: cmp x4, x2 0.00 : ffff80001011d038: b.ne ffff80001011cfc8 // b.any : cfs_rq->next = NULL; 0.00 : ffff80001011d03c: str xzr, [x3, #80] : for_each_sched_entity(se) { 0.00 : ffff80001011d040: ldr x2, [x2, #120] 0.00 : ffff80001011d044: cbnz x2, ffff80001011d02c 0.00 : ffff80001011d048: b ffff80001011cfc8 : __clear_buddies_last(): : for_each_sched_entity(se) { 0.00 : ffff80001011d04c: cbz x1, ffff80001011cfbc : struct cfs_rq *cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011d050: ldr x3, [x1, #128] : if (cfs_rq->last != se) 0.00 : ffff80001011d054: ldr x2, [x3, #88] 0.00 : ffff80001011d058: cmp x1, x2 0.00 : ffff80001011d05c: b.ne ffff80001011cfbc // b.any 0.00 : ffff80001011d060: mov x2, x1 0.00 : ffff80001011d064: b ffff80001011d078 : struct cfs_rq *cfs_rq = cfs_rq_of(se); 0.00 : ffff80001011d068: ldr x3, [x2, #128] : if (cfs_rq->last != se) 0.00 : ffff80001011d06c: ldr x4, [x3, #88] 0.00 : ffff80001011d070: cmp x4, x2 0.00 : ffff80001011d074: b.ne ffff80001011cfbc // b.any : cfs_rq->last = NULL; 0.00 : ffff80001011d078: str xzr, [x3, #88] : for_each_sched_entity(se) { 0.00 : ffff80001011d07c: ldr x2, [x2, #120] 0.00 : ffff80001011d080: cbnz x2, ffff80001011d068 0.00 : ffff80001011d084: b ffff80001011cfbc Percent | Source code & Disassembly of vmlinux for cycles (32 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001010bce8 : : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 15.53 : ffff80001010bce8: mrs x0, sp_el0 : kthread_blkcg(): : */ : struct cgroup_subsys_state *kthread_blkcg(void) : { : struct kthread *kthread; : : if (current->flags & PF_KTHREAD) { 12.55 : ffff80001010bcec: ldr w1, [x0, #44] 0.00 : ffff80001010bcf0: tbz w1, #21, ffff80001010bd04 : to_kthread(): : return (__force void *)k->set_child_tid; 0.00 : ffff80001010bcf4: ldr x0, [x0, #1344] : kthread_blkcg(): : kthread = to_kthread(current); : if (kthread) 0.00 : ffff80001010bcf8: cbz x0, ffff80001010bd04 : return kthread->blkcg_css; 0.00 : ffff80001010bcfc: ldr x0, [x0, #88] : } : return NULL; : } 0.00 : ffff80001010bd00: ret : return NULL; 71.92 : ffff80001010bd04: mov x0, #0x0 // #0 : } 0.00 : ffff80001010bd08: ret Percent | Source code & Disassembly of vmlinux for cycles (15 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010121528 : : check_preempt_wakeup(): : : /* : * Preempt the current task with a newly woken task if needed: : */ : static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) : { 32.62 : ffff800010121528: stp x29, x30, [sp, #-64]! 0.00 : ffff80001012152c: mov x29, sp 0.00 : ffff800010121530: stp x19, x20, [sp, #16] : struct task_struct *curr = rq->curr; : struct sched_entity *se = &curr->se, *pse = &p->se; 0.00 : ffff800010121534: add x19, x1, #0xc0 : { 0.00 : ffff800010121538: str x23, [sp, #48] : struct task_struct *curr = rq->curr; 6.58 : ffff80001012153c: ldr x23, [x0, #2352] : struct sched_entity *se = &curr->se, *pse = &p->se; 0.00 : ffff800010121540: add x20, x23, #0xc0 : struct cfs_rq *cfs_rq = task_cfs_rq(curr); : int scale = cfs_rq->nr_running >= sched_nr_latency; : int next_buddy_marked = 0; : : if (unlikely(se == pse)) 0.00 : ffff800010121544: cmp x20, x19 0.00 : ffff800010121548: b.eq ffff8000101215fc // b.none : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001012154c: ldr x2, [x23] : check_preempt_wakeup(): : * group (e.g. via set_curr_task), since update_curr() (in the : * enqueue of curr) will have resulted in resched being set. This : * prevents us from potentially nominating it as a false LAST_BUDDY : * below. : */ : if (test_tsk_need_resched(curr)) 0.00 : ffff800010121550: tst w2, #0x2 0.00 : ffff800010121554: b.ne ffff8000101215fc // b.any 0.00 : ffff800010121558: stp x21, x22, [x29, #32] 0.00 : ffff80001012155c: mov x21, x0 : task_cfs_rq(): : return p->se.cfs_rq; 0.00 : ffff800010121560: ldr x0, [x23, #320] : check_preempt_wakeup(): : return; : : /* Idle tasks are by definition preempted by non-idle tasks. */ : if (unlikely(task_has_idle_policy(curr)) && 0.00 : ffff800010121564: ldr w2, [x23, #728] 0.00 : ffff800010121568: ldr w1, [x1, #728] 0.00 : ffff80001012156c: cmp w2, #0x5 : int scale = cfs_rq->nr_running >= sched_nr_latency; 0.00 : ffff800010121570: ldr w22, [x0, #24] : if (unlikely(task_has_idle_policy(curr)) && 0.00 : ffff800010121574: b.eq ffff80001012169c // b.none : : /* : * Batch and idle tasks do not preempt non-idle tasks (their preemption : * is driven by the tick): : */ : if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION)) 0.00 : ffff800010121578: cbnz w1, ffff8000101215f8 : find_matching_se(): : pse_depth = (*pse)->depth; 0.00 : ffff80001012157c: ldr w2, [x19, #112] : se_depth = (*se)->depth; 0.00 : ffff800010121580: ldr w1, [x20, #112] : while (se_depth > pse_depth) { 0.00 : ffff800010121584: cmp w1, w2 0.00 : ffff800010121588: b.le ffff800010121684 0.00 : ffff80001012158c: nop : se_depth--; 0.00 : ffff800010121590: sub w1, w1, #0x1 : *se = parent_entity(*se); 0.00 : ffff800010121594: ldr x20, [x20, #120] : while (se_depth > pse_depth) { 0.00 : ffff800010121598: cmp w2, w1 0.00 : ffff80001012159c: b.ne ffff800010121590 // b.any 0.00 : ffff8000101215a0: ldr x0, [x20, #128] : while (!is_same_group(*se, *pse)) { 0.00 : ffff8000101215a4: ldr x1, [x19, #128] 0.00 : ffff8000101215a8: cmp x0, #0x0 0.00 : ffff8000101215ac: ccmp x1, x0, #0x0, ne // ne = any 0.00 : ffff8000101215b0: b.eq ffff8000101215dc // b.none 0.00 : ffff8000101215b4: nop : *pse = parent_entity(*pse); 0.00 : ffff8000101215b8: ldr x19, [x19, #120] : *se = parent_entity(*se); 0.00 : ffff8000101215bc: ldr x20, [x20, #120] : is_same_group(): : if (se->cfs_rq == pse->cfs_rq) 0.00 : ffff8000101215c0: ldr x1, [x19, #128] : find_matching_se(): : while (!is_same_group(*se, *pse)) { 0.00 : ffff8000101215c4: ldr x0, [x20, #128] : is_same_group(): : if (se->cfs_rq == pse->cfs_rq) 0.00 : ffff8000101215c8: cmp x1, x0 0.00 : ffff8000101215cc: cset w1, ne // ne = any : find_matching_se(): : while (!is_same_group(*se, *pse)) { 0.00 : ffff8000101215d0: cmp x0, #0x0 0.00 : ffff8000101215d4: csinc w1, w1, wzr, ne // ne = any 0.00 : ffff8000101215d8: cbnz w1, ffff8000101215b8 : check_preempt_wakeup(): : return; : : find_matching_se(&se, &pse); : update_curr(cfs_rq_of(se)); 0.00 : ffff8000101215dc: bl ffff80001011fab8 : BUG_ON(!pse); 0.00 : ffff8000101215e0: cbz x19, ffff8000101216a8 : if (wakeup_preempt_entity(se, pse) == 1) { 0.00 : ffff8000101215e4: ldr x0, [x20, #88] 0.00 : ffff8000101215e8: mov x1, x19 0.00 : ffff8000101215ec: bl ffff80001011dd70 0.00 : ffff8000101215f0: cmp w0, #0x1 0.00 : ffff8000101215f4: b.eq ffff80001012160c // b.none 0.00 : ffff8000101215f8: ldp x21, x22, [x29, #32] : if (unlikely(!se->on_rq || curr == rq->idle)) : return; : : if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) : set_last_buddy(se); : } 13.43 : ffff8000101215fc: ldp x19, x20, [sp, #16] 47.38 : ffff800010121600: ldr x23, [sp, #48] 0.00 : ffff800010121604: ldp x29, x30, [sp], #64 0.00 : ffff800010121608: ret : set_next_buddy(): : if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 0.00 : ffff80001012160c: ldr x0, [x19, #136] 0.00 : ffff800010121610: cbz x0, ffff800010121674 0.00 : ffff800010121614: nop : cfs_rq_of(se)->next = se; 0.00 : ffff800010121618: ldr x0, [x19, #128] 0.00 : ffff80001012161c: str x19, [x0, #80] : for_each_sched_entity(se) { 0.00 : ffff800010121620: ldr x19, [x19, #120] 0.00 : ffff800010121624: cbnz x19, ffff800010121618 : check_preempt_wakeup(): : resched_curr(rq); 0.00 : ffff800010121628: mov x0, x21 0.00 : ffff80001012162c: bl ffff800010114bb8 : if (unlikely(!se->on_rq || curr == rq->idle)) 0.00 : ffff800010121630: ldr w0, [x20, #64] 0.00 : ffff800010121634: cbz w0, ffff8000101215f8 : if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) 0.00 : ffff800010121638: ldr x0, [x21, #2360] 0.00 : ffff80001012163c: cmp x0, x23 0.00 : ffff800010121640: ccmp w22, #0x7, #0x0, ne // ne = any 0.00 : ffff800010121644: b.ls ffff8000101215f8 // b.plast 0.00 : ffff800010121648: ldr x0, [x20, #136] 0.00 : ffff80001012164c: cbnz x0, ffff8000101215f8 : set_last_buddy(): : if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 0.00 : ffff800010121650: ldr w0, [x20, #536] 0.00 : ffff800010121654: cmp w0, #0x5 0.00 : ffff800010121658: b.eq ffff8000101215f8 // b.none 0.00 : ffff80001012165c: nop : for_each_sched_entity(se) { 0.00 : ffff800010121660: cbz x20, ffff8000101215f8 : cfs_rq_of(se)->last = se; 0.00 : ffff800010121664: ldr x0, [x20, #128] 0.00 : ffff800010121668: str x20, [x0, #88] : for_each_sched_entity(se) { 0.00 : ffff80001012166c: ldr x20, [x20, #120] 0.00 : ffff800010121670: b ffff800010121660 : set_next_buddy(): : if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 0.00 : ffff800010121674: ldr w0, [x19, #536] 0.00 : ffff800010121678: cmp w0, #0x5 0.00 : ffff80001012167c: b.ne ffff800010121618 // b.any 0.00 : ffff800010121680: b ffff800010121628 : find_matching_se(): : while (pse_depth > se_depth) { 0.00 : ffff800010121684: b.ge ffff8000101215a4 // b.tcont : pse_depth--; 0.00 : ffff800010121688: sub w2, w2, #0x1 : *pse = parent_entity(*pse); 0.00 : ffff80001012168c: ldr x19, [x19, #120] : while (pse_depth > se_depth) { 0.00 : ffff800010121690: cmp w1, w2 0.00 : ffff800010121694: b.ne ffff800010121688 // b.any 0.00 : ffff800010121698: b ffff8000101215a4 : check_preempt_wakeup(): : if (unlikely(task_has_idle_policy(curr)) && 0.00 : ffff80001012169c: cmp w1, #0x5 0.00 : ffff8000101216a0: b.ne ffff800010121628 // b.any 0.00 : ffff8000101216a4: b ffff8000101215f8 : BUG_ON(!pse); 0.00 : ffff8000101216a8: brk #0x800 Percent | Source code & Disassembly of vmlinux for cycles (23 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016acf8 : : hrtimer_forward(): : u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) : { : u64 orun = 1; : ktime_t delta; : : delta = ktime_sub(now, hrtimer_get_expires(timer)); 53.36 : ffff80001016acf8: ldr x3, [x0, #24] : { 0.00 : ffff80001016acfc: mov x4, x0 : : if (delta < 0) 0.00 : ffff80001016ad00: subs x0, x1, x3 0.00 : ffff80001016ad04: b.mi ffff80001016ad78 // b.first : return 0; : : if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED)) 0.00 : ffff80001016ad08: ldrb w5, [x4, #56] 0.00 : ffff80001016ad0c: tbnz w5, #0, ffff80001016ad80 : return 0; : : if (interval < hrtimer_resolution) 4.32 : ffff80001016ad10: adrp x5, ffff80001190a000 <__per_cpu_offset+0x6d8> 8.68 : ffff80001016ad14: ldr w5, [x5, #700] 0.00 : ffff80001016ad18: cmp x5, x2 0.00 : ffff80001016ad1c: b.gt ffff80001016ad60 : interval = hrtimer_resolution; : : if (unlikely(delta >= interval)) { 0.00 : ffff80001016ad20: cmp x0, x2 0.00 : ffff80001016ad24: b.ge ffff80001016ad8c // b.tcont 0.00 : ffff80001016ad28: ldr x6, [x4, #32] 0.00 : ffff80001016ad2c: mov x5, x2 : u64 orun = 1; 0.00 : ffff80001016ad30: mov x0, #0x1 // #1 : ktime_add_safe(): : if (res < 0 || res < lhs || res < rhs) 0.00 : ffff80001016ad34: adds x1, x3, x2 : res = ktime_set(KTIME_SEC_MAX, 0); 9.07 : ffff80001016ad38: mov x7, #0x7fffffffffffffff // #9223372036854775807 : if (res < 0 || res < lhs || res < rhs) 0.00 : ffff80001016ad3c: ccmp x1, x3, #0x1, pl // pl = nfrst : res = ktime_set(KTIME_SEC_MAX, 0); 0.00 : ffff80001016ad40: ccmp x1, x5, #0x1, ge // ge = tcont 0.00 : ffff80001016ad44: csel x1, x1, x7, ge // ge = tcont : if (res < 0 || res < lhs || res < rhs) 0.00 : ffff80001016ad48: adds x2, x6, x2 0.00 : ffff80001016ad4c: ccmp x2, x6, #0x1, pl // pl = nfrst : res = ktime_set(KTIME_SEC_MAX, 0); 0.00 : ffff80001016ad50: ccmp x2, x5, #0x1, ge // ge = tcont 0.00 : ffff80001016ad54: csel x2, x2, x7, ge // ge = tcont : hrtimer_add_expires(): : } : : static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time) : { : timer->node.expires = ktime_add_safe(timer->node.expires, time); : timer->_softexpires = ktime_add_safe(timer->_softexpires, time); 0.00 : ffff80001016ad58: stp x1, x2, [x4, #24] : hrtimer_forward(): : orun++; : } : hrtimer_add_expires(timer, interval); : : return orun; : } 0.00 : ffff80001016ad5c: ret 0.00 : ffff80001016ad60: mov x2, x5 : if (unlikely(delta >= interval)) { 0.00 : ffff80001016ad64: cmp x5, x0 0.00 : ffff80001016ad68: ldr x6, [x4, #32] 0.00 : ffff80001016ad6c: b.le ffff80001016ad98 : u64 orun = 1; 0.00 : ffff80001016ad70: mov x0, #0x1 // #1 0.00 : ffff80001016ad74: b ffff80001016ad34 : return 0; 24.57 : ffff80001016ad78: mov x0, #0x0 // #0 : } 0.00 : ffff80001016ad7c: ret : if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED)) 0.00 : ffff80001016ad80: brk #0x800 : return 0; 0.00 : ffff80001016ad84: mov x0, #0x0 // #0 : } 0.00 : ffff80001016ad88: ret : ktime_divns(): : { : /* : * 32-bit implementation cannot handle negative divisors, : * so catch them on 64bit as well. : */ : WARN_ON(div < 0); 0.00 : ffff80001016ad8c: tbnz x2, #63, ffff80001016adbc 0.00 : ffff80001016ad90: ldr x6, [x4, #32] 0.00 : ffff80001016ad94: mov x5, x2 : return kt / div; 0.00 : ffff80001016ad98: sdiv x0, x0, x5 : hrtimer_forward(): : hrtimer_add_expires_ns(timer, incr * orun); 0.00 : ffff80001016ad9c: mul x7, x0, x2 : hrtimer_add_expires_ns(): : } : : static inline void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns) : { : timer->node.expires = ktime_add_ns(timer->node.expires, ns); 0.00 : ffff80001016ada0: add x3, x3, x7 : timer->_softexpires = ktime_add_ns(timer->_softexpires, ns); 0.00 : ffff80001016ada4: add x6, x6, x7 0.00 : ffff80001016ada8: stp x3, x6, [x4, #24] : hrtimer_forward(): : if (hrtimer_get_expires_tv64(timer) > now) 0.00 : ffff80001016adac: cmp x1, x3 0.00 : ffff80001016adb0: b.lt ffff80001016ad5c // b.tstop : orun++; 0.00 : ffff80001016adb4: add x0, x0, #0x1 0.00 : ffff80001016adb8: b ffff80001016ad34 : ktime_divns(): : WARN_ON(div < 0); 0.00 : ffff80001016adbc: brk #0x800 0.00 : ffff80001016adc0: ldr x3, [x4, #24] 0.00 : ffff80001016adc4: b ffff80001016ad90 Percent | Source code & Disassembly of vmlinux for cycles (35 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d9be0 : : do_io_getevents(): : static long do_io_getevents(aio_context_t ctx_id, : long min_nr, : long nr, : struct io_event __user *events, : struct timespec64 *ts) : { 0.00 : ffff8000102d9be0: stp x29, x30, [sp, #-64]! 0.00 : ffff8000102d9be4: mov x29, sp 5.74 : ffff8000102d9be8: stp x19, x20, [sp, #16] 0.00 : ffff8000102d9bec: mov x20, x1 0.00 : ffff8000102d9bf0: stp x21, x22, [sp, #32] : ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX; 0.00 : ffff8000102d9bf4: mov x19, #0x7fffffffffffffff // #9223372036854775807 : { 0.00 : ffff8000102d9bf8: str x23, [sp, #48] 0.00 : ffff8000102d9bfc: mov x22, x2 0.00 : ffff8000102d9c00: mov x23, x3 : ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX; 0.00 : ffff8000102d9c04: cbz x4, ffff8000102d9c2c 0.00 : ffff8000102d9c08: ldp x1, x3, [x4] : ktime_set(): : * : * Return: The ktime_t representation of the value. : */ : static inline ktime_t ktime_set(const s64 secs, const unsigned long nsecs) : { : if (unlikely(secs >= KTIME_SEC_MAX)) 0.00 : ffff8000102d9c0c: mov x2, #0x7d03 // #32003 0.00 : ffff8000102d9c10: movk x2, #0x25c1, lsl #16 0.00 : ffff8000102d9c14: movk x2, #0x2, lsl #32 0.00 : ffff8000102d9c18: cmp x1, x2 0.00 : ffff8000102d9c1c: b.gt ffff8000102d9c2c : return KTIME_MAX; : : return secs * NSEC_PER_SEC + (s64)nsecs; 0.00 : ffff8000102d9c20: mov x19, #0xca00 // #51712 0.00 : ffff8000102d9c24: movk x19, #0x3b9a, lsl #16 0.00 : ffff8000102d9c28: madd x19, x1, x19, x3 : do_io_getevents(): : struct kioctx *ioctx = lookup_ioctx(ctx_id); 11.30 : ffff8000102d9c2c: bl ffff8000102d95e8 0.00 : ffff8000102d9c30: mov x21, x0 : long ret = -EINVAL; : : if (likely(ioctx)) { 0.00 : ffff8000102d9c34: cbz x0, ffff8000102d9cd8 : if (likely(min_nr <= nr && min_nr >= 0)) 0.00 : ffff8000102d9c38: cmp x20, #0x0 0.00 : ffff8000102d9c3c: ccmp x20, x22, #0x0, ge // ge = tcont 0.00 : ffff8000102d9c40: b.gt ffff8000102d9cf4 : ret = read_events(ioctx, min_nr, nr, events, until); 5.44 : ffff8000102d9c44: mov x4, x19 0.00 : ffff8000102d9c48: mov x3, x23 0.00 : ffff8000102d9c4c: mov x2, x22 0.00 : ffff8000102d9c50: mov x1, x20 0.00 : ffff8000102d9c54: bl ffff8000102d9490 0.00 : ffff8000102d9c58: mov x19, x0 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000102d9c5c: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000102d9c60: ldr x0, [x21, #8] : __ref_is_percpu(): : * Theoretically, the following could test just ATOMIC; however, : * then we'd have to mask off DEAD separately as DEAD may be : * visible without ATOMIC if we race with percpu_ref_kill(). DEAD : * implies ATOMIC anyway. Test them together. : */ : if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD)) 0.00 : ffff8000102d9c64: tst x0, #0x3 0.00 : ffff8000102d9c68: b.ne ffff8000102d9cfc // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 2.90 : ffff8000102d9c6c: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000102d9c70: ldr w2, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000102d9c74: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102d9c78: str w2, [x1, #16] : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102d9c7c: mov x3, #0xffffffffffffffff // #-1 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000102d9c80: mrs x2, tpidr_el1 : __percpu_add_case_64(): : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102d9c84: add x0, x0, x2 0.00 : ffff8000102d9c88: ldxr x5, [x0] 31.47 : ffff8000102d9c8c: add x5, x5, x3 0.00 : ffff8000102d9c90: stxr w4, x5, [x0] 0.00 : ffff8000102d9c94: cbnz w4, ffff8000102d9c88 : __read_once_size(): : __READ_ONCE_SIZE; 2.89 : ffff8000102d9c98: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102d9c9c: add x0, x0, x3 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 5.77 : ffff8000102d9ca0: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102d9ca4: cbnz x0, ffff8000102d9cc8 : percpu_ref_put_many(): : unsigned long __percpu *percpu_count; : : rcu_read_lock(); : : if (__ref_is_percpu(ref, &percpu_count)) : this_cpu_sub(*percpu_count, nr); 0.00 : ffff8000102d9ca8: bl ffff800010cd2e78 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 31.61 : ffff8000102d9cac: bl ffff800010160d20 <__rcu_read_unlock> : do_io_getevents(): : percpu_ref_put(&ioctx->users); : } : : return ret; : } 0.00 : ffff8000102d9cb0: ldr x23, [sp, #48] 0.00 : ffff8000102d9cb4: mov x0, x19 2.87 : ffff8000102d9cb8: ldp x19, x20, [sp, #16] 0.00 : ffff8000102d9cbc: ldp x21, x22, [sp, #32] 0.00 : ffff8000102d9cc0: ldp x29, x30, [sp], #64 0.00 : ffff8000102d9cc4: ret : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102d9cc8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000102d9ccc: cbnz x0, ffff8000102d9cac : percpu_ref_put_many(): 0.00 : ffff8000102d9cd0: bl ffff800010cd2e78 0.00 : ffff8000102d9cd4: b ffff8000102d9cac : do_io_getevents(): : long ret = -EINVAL; 0.00 : ffff8000102d9cd8: mov x19, #0xffffffffffffffea // #-22 : } 0.00 : ffff8000102d9cdc: ldr x23, [sp, #48] 0.00 : ffff8000102d9ce0: mov x0, x19 0.00 : ffff8000102d9ce4: ldp x19, x20, [sp, #16] 0.00 : ffff8000102d9ce8: ldp x21, x22, [sp, #32] 0.00 : ffff8000102d9cec: ldp x29, x30, [sp], #64 0.00 : ffff8000102d9cf0: ret : long ret = -EINVAL; 0.00 : ffff8000102d9cf4: mov x19, #0xffffffffffffffea // #-22 0.00 : ffff8000102d9cf8: b ffff8000102d9c5c : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102d9cfc: b ffff8000102d9d28 0.00 : ffff8000102d9d00: b ffff8000102d9d28 : __lse_atomic64_sub_return(): : } : : ATOMIC64_OP_SUB_RETURN(_relaxed, ) : ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") : ATOMIC64_OP_SUB_RETURN(_release, l, "memory") : ATOMIC64_OP_SUB_RETURN( , al, "memory") 0.00 : ffff8000102d9d04: mov x0, #0x1 // #1 0.00 : ffff8000102d9d08: neg x0, x0 0.00 : ffff8000102d9d0c: ldaddal x0, x1, [x21] 0.00 : ffff8000102d9d10: add x0, x0, x1 : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000102d9d14: cbnz x0, ffff8000102d9cac : ref->release(ref); 0.00 : ffff8000102d9d18: ldr x1, [x21, #16] 0.00 : ffff8000102d9d1c: mov x0, x21 0.00 : ffff8000102d9d20: blr x1 0.00 : ffff8000102d9d24: b ffff8000102d9cac : __ll_sc_atomic64_sub_return(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) : ATOMIC64_OPS(sub, sub, J) 0.00 : ffff8000102d9d28: mov x1, #0x1 // #1 0.00 : ffff8000102d9d2c: b ffff8000102dd948 <__arm64_compat_sys_io_pgetevents_time64+0x2d8> : percpu_ref_put_many(): : else if (unlikely(atomic_long_sub_and_test(nr, &ref->count))) 0.00 : ffff8000102d9d30: cbnz x0, ffff8000102d9cac 0.00 : ffff8000102d9d34: b ffff8000102d9d18 Percent | Source code & Disassembly of vmlinux for cycles (48 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010476d18 : : blk_queue_bounce(): : mempool_t *pool; : : /* : * Data-less bio, nothing to bounce : */ : if (!bio_has_data(*bio_orig)) 2.09 : ffff800010476d18: ldr x3, [x1] : bio_has_data(): : /* : * Check whether this bio carries any data or not. A NULL bio is allowed. : */ : static inline bool bio_has_data(struct bio *bio) : { : if (bio && 0.00 : ffff800010476d1c: cbz x3, ffff800010476d58 : bio->bi_iter.bi_size && 4.15 : ffff800010476d20: ldr w6, [x3, #40] : if (bio && 0.00 : ffff800010476d24: cbz w6, ffff800010476d58 : blk_queue_bounce(): : { 10.15 : ffff800010476d28: stp x29, x30, [sp, #-112]! 0.00 : ffff800010476d2c: mov x29, sp 0.00 : ffff800010476d30: str x23, [sp, #48] : bio_has_data(): : bio_op(bio) != REQ_OP_DISCARD && 0.00 : ffff800010476d34: ldr w23, [x3, #16] 0.00 : ffff800010476d38: and w2, w23, #0xff : bio->bi_iter.bi_size && 0.00 : ffff800010476d3c: sub w4, w2, #0x3 : bio_op(bio) != REQ_OP_SECURE_ERASE && 0.00 : ffff800010476d40: tst w4, #0xfffffffd 0.00 : ffff800010476d44: ccmp w2, #0x9, #0x4, ne // ne = any 0.00 : ffff800010476d48: b.ne ffff800010476d5c // b.any : blk_queue_bounce(): : : /* : * slow path : */ : __blk_queue_bounce(q, bio_orig, pool); : } 2.07 : ffff800010476d4c: ldr x23, [sp, #48] 0.00 : ffff800010476d50: ldp x29, x30, [sp], #112 0.00 : ffff800010476d54: ret 0.00 : ffff800010476d58: ret 0.00 : ffff800010476d5c: stp x20, x21, [x29, #24] : if (!(q->bounce_gfp & GFP_DMA)) { 10.36 : ffff800010476d60: ldr w4, [x0, #120] 0.00 : ffff800010476d64: tbnz w4, #0, ffff800010476da0 : if (q->limits.bounce_pfn >= blk_max_pfn) 4.25 : ffff800010476d68: adrp x4, ffff800011b3a000 4.20 : ffff800010476d6c: ldr x5, [x0, #1040] 0.00 : ffff800010476d70: ldr x4, [x4, #1120] 0.00 : ffff800010476d74: cmp x5, x4 0.00 : ffff800010476d78: b.cs ffff8000104770bc // b.hs, b.nlast : pool = &page_pool; 0.00 : ffff800010476d7c: adrp x20, ffff800011b3a000 0.00 : ffff800010476d80: add x21, x20, #0xc90 0.00 : ffff800010476d84: add x21, x21, #0x48 0.00 : ffff800010476d88: str x19, [x29, #16] 0.00 : ffff800010476d8c: str x22, [x29, #40] 0.00 : ffff800010476d90: stp x24, x25, [x29, #56] 0.00 : ffff800010476d94: stp x26, x27, [x29, #72] 0.00 : ffff800010476d98: str x28, [x29, #88] 0.00 : ffff800010476d9c: b ffff800010476dc8 : BUG_ON(!mempool_initialized(&isa_page_pool)); 0.00 : ffff800010476da0: adrp x20, ffff800011b3a000 0.00 : ffff800010476da4: add x4, x20, #0xc90 : pool = &isa_page_pool; 0.00 : ffff800010476da8: mov x21, x4 : BUG_ON(!mempool_initialized(&isa_page_pool)); 0.00 : ffff800010476dac: ldr x4, [x4, #16] 0.00 : ffff800010476db0: str x19, [x29, #16] 0.00 : ffff800010476db4: str x22, [x29, #40] 0.00 : ffff800010476db8: stp x24, x25, [x29, #56] 0.00 : ffff800010476dbc: stp x26, x27, [x29, #72] 0.00 : ffff800010476dc0: str x28, [x29, #88] 0.00 : ffff800010476dc4: cbz x4, ffff8000104773b8 0.00 : ffff800010476dc8: ldr w10, [x3, #16] 0.00 : ffff800010476dcc: mov x22, x0 : bio_is_passthrough(): : : static inline bool bio_is_passthrough(struct bio *bio) : { : unsigned op = bio_op(bio); : : return blk_op_is_scsi(op) || blk_op_is_private(op); 0.00 : ffff800010476dd0: sub w2, w2, #0x20 : bio_no_advance_iter(): : return false; : } : : static inline bool bio_no_advance_iter(struct bio *bio) : { : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff800010476dd4: mov w0, #0xfb // #251 0.00 : ffff800010476dd8: and w9, w10, #0xff : bio_is_passthrough(): 0.00 : ffff800010476ddc: cmp w2, #0x3 : bio_no_advance_iter(): 0.00 : ffff800010476de0: sub w9, w9, #0x5 0.00 : ffff800010476de4: mov x19, x1 0.00 : ffff800010476de8: and w10, w10, w0 : bio_is_passthrough(): 0.00 : ffff800010476dec: cset w26, ls // ls = plast : __blk_queue_bounce(): : bool bounce = false; 0.00 : ffff800010476df0: mov w25, #0x0 // #0 : int sectors = 0; 0.00 : ffff800010476df4: mov w1, #0x0 // #0 0.00 : ffff800010476df8: adrp x24, ffff8000112f2000 : bio_for_each_segment(from, *bio_orig, iter) { 0.00 : ffff800010476dfc: mov w13, #0x1000 // #4096 : bounce = true; 0.00 : ffff800010476e00: mov w11, #0x1 // #1 : bio_for_each_segment(from, *bio_orig, iter) { 0.00 : ffff800010476e04: mov w0, #0x0 // #0 0.00 : ffff800010476e08: ldp w7, w2, [x3, #44] : bio_no_advance_iter(): 0.00 : ffff800010476e0c: and w9, w9, #0xfffffffb 0.00 : ffff800010476e10: ldr x8, [x3, #104] 0.00 : ffff800010476e14: ldr x12, [x22, #1040] 0.00 : ffff800010476e18: b ffff800010476e28 : bio_advance_iter(): : unsigned bytes) : { : iter->bi_sector += bytes >> 9; : : if (bio_no_advance_iter(bio)) : iter->bi_size -= bytes; 0.00 : ffff800010476e1c: sub w6, w6, w4 0.00 : ffff800010476e20: add w0, w0, #0x1 : __blk_queue_bounce(): 0.00 : ffff800010476e24: cbz w6, ffff800010476ef0 0.00 : ffff800010476e28: ubfiz x4, x7, #4, #32 : if (page_to_pfn(from.bv_page) > q->limits.bounce_pfn) 0.00 : ffff800010476e2c: ldr x15, [x24, #1880] : bio_for_each_segment(from, *bio_orig, iter) { 0.00 : ffff800010476e30: add x14, x8, x4 0.00 : ffff800010476e34: ldr x16, [x8, x4] 0.00 : ffff800010476e38: ldp w5, w3, [x14, #8] 0.00 : ffff800010476e3c: add w3, w2, w3 0.00 : ffff800010476e40: sub w4, w5, w2 0.00 : ffff800010476e44: cmp w4, w6 0.00 : ffff800010476e48: and w14, w3, #0xfff 0.00 : ffff800010476e4c: sub w14, w13, w14 0.00 : ffff800010476e50: csel w4, w4, w6, ls // ls = plast 0.00 : ffff800010476e54: lsr w3, w3, #12 0.00 : ffff800010476e58: cmp w4, w14 0.00 : ffff800010476e5c: csel w4, w4, w14, ls // ls = plast : sectors += from.bv_len >> 9; 0.00 : ffff800010476e60: cmp w0, #0x100 : bio_for_each_segment(from, *bio_orig, iter) { 0.00 : ffff800010476e64: add x3, x16, x3, lsl #6 : if (page_to_pfn(from.bv_page) > q->limits.bounce_pfn) 0.00 : ffff800010476e68: sub x3, x3, x15 : sectors += from.bv_len >> 9; 0.00 : ffff800010476e6c: add w14, w1, w4, lsr #9 0.00 : ffff800010476e70: csel w1, w14, w1, cc // cc = lo, ul, last : bounce = true; 0.00 : ffff800010476e74: cmp x12, x3, asr #6 0.00 : ffff800010476e78: csel w25, w25, w11, cs // cs = hs, nlast : bio_no_advance_iter(): : bio_op(bio) == REQ_OP_WRITE_SAME || 0.00 : ffff800010476e7c: cmp w9, #0x0 0.00 : ffff800010476e80: ccmp w10, #0x3, #0x4, ne // ne = any 0.00 : ffff800010476e84: b.eq ffff800010476e1c // b.none : bvec_iter_advance(): : static inline bool bvec_iter_advance(const struct bio_vec *bv, : struct bvec_iter *iter, unsigned bytes) : { : unsigned int idx = iter->bi_idx; : : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff800010476e88: cmp w4, w6 0.00 : ffff800010476e8c: b.hi ffff800010476ec4 // b.pmore : "Attempted to advance past end of bvec iter\n")) { : iter->bi_size = 0; : return false; : } : : iter->bi_size -= bytes; 0.00 : ffff800010476e90: sub w6, w6, w4 : bytes += iter->bi_bvec_done; : : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010476e94: adds w2, w2, w4 0.00 : ffff800010476e98: b.ne ffff800010476eb8 // b.any 0.00 : ffff800010476e9c: b ffff800010476e20 : bytes -= bv[idx].bv_len; : idx++; 0.00 : ffff800010476ea0: add w7, w7, #0x1 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010476ea4: subs w2, w2, w5 0.00 : ffff800010476ea8: b.eq ffff800010476e20 // b.none 0.00 : ffff800010476eac: ubfiz x3, x7, #4, #32 0.00 : ffff800010476eb0: add x3, x8, x3 0.00 : ffff800010476eb4: ldr w5, [x3, #8] 0.00 : ffff800010476eb8: cmp w5, w2 0.00 : ffff800010476ebc: b.ls ffff800010476ea0 // b.plast 0.00 : ffff800010476ec0: b ffff800010476e20 : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff800010476ec4: adrp x2, ffff800011ad8000 0.00 : ffff800010476ec8: ldrb w0, [x2, #1296] 0.00 : ffff800010476ecc: cbnz w0, ffff800010476ef0 0.00 : ffff800010476ed0: mov w3, #0x1 // #1 0.00 : ffff800010476ed4: str w1, [x29, #104] 0.00 : ffff800010476ed8: strb w3, [x2, #1296] 0.00 : ffff800010476edc: adrp x0, ffff8000111e9000 0.00 : ffff800010476ee0: add x0, x0, #0x680 0.00 : ffff800010476ee4: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010476ee8: brk #0x800 0.00 : ffff800010476eec: ldr w1, [x29, #104] : __blk_queue_bounce(): : if (!bounce) 0.00 : ffff800010476ef0: cbz w25, ffff8000104770a4 : bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL : 0.00 : ffff800010476ef4: mov x2, #0x0 // #0 0.00 : ffff800010476ef8: ldr x27, [x19] : if (!passthrough && sectors < bio_sectors(*bio_orig)) { 0.00 : ffff800010476efc: cbnz w26, ffff800010476f14 0.00 : ffff800010476f00: ldr w0, [x27, #40] 0.00 : ffff800010476f04: cmp w1, w0, lsr #9 0.00 : ffff800010476f08: b.cc ffff800010477388 // b.lo, b.ul, b.last : bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL : 0.00 : ffff800010476f0c: add x2, x20, #0xc90 0.00 : ffff800010476f10: add x2, x2, #0x98 : bio_segments(): : /* : * We special case discard/write same/write zeroes, because they : * interpret bi_size differently: : */ : : switch (bio_op(bio)) { 0.00 : ffff800010476f14: ldrb w0, [x27, #16] 0.00 : ffff800010476f18: cmp w0, #0x5 0.00 : ffff800010476f1c: b.eq ffff8000104771a0 // b.none 0.00 : ffff800010476f20: b.ls ffff8000104771c8 // b.plast : case REQ_OP_DISCARD: : case REQ_OP_SECURE_ERASE: : case REQ_OP_WRITE_ZEROES: : return 0; : case REQ_OP_WRITE_SAME: : return 1; 0.00 : ffff800010476f24: mov w1, #0x1 // #1 : switch (bio_op(bio)) { 0.00 : ffff800010476f28: cmp w0, #0x7 0.00 : ffff800010476f2c: b.ne ffff800010477198 // b.any : bounce_clone_bio(): : bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs); 0.00 : ffff800010476f30: mov w0, #0xc00 // #3072 0.00 : ffff800010476f34: bl ffff800010456f78 0.00 : ffff800010476f38: mov x25, x0 : if (!bio) 0.00 : ffff800010476f3c: cbz x0, ffff800010477004 : bio->bi_disk = bio_src->bi_disk; 0.00 : ffff800010476f40: ldr x0, [x27, #8] 0.00 : ffff800010476f44: str x0, [x25, #8] : bio->bi_opf = bio_src->bi_opf; 0.00 : ffff800010476f48: ldr w0, [x27, #16] 0.00 : ffff800010476f4c: str w0, [x25, #16] : switch (bio_op(bio)) { 0.00 : ffff800010476f50: and w0, w0, #0xff : bio->bi_ioprio = bio_src->bi_ioprio; 0.00 : ffff800010476f54: ldrh w1, [x27, #22] : switch (bio_op(bio)) { 0.00 : ffff800010476f58: cmp w0, #0x5 : bio->bi_ioprio = bio_src->bi_ioprio; 0.00 : ffff800010476f5c: strh w1, [x25, #22] : bio->bi_write_hint = bio_src->bi_write_hint; 0.00 : ffff800010476f60: ldrh w1, [x27, #24] 0.00 : ffff800010476f64: strh w1, [x25, #24] : bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; 0.00 : ffff800010476f68: ldr x1, [x27, #32] 0.00 : ffff800010476f6c: str x1, [x25, #32] : bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; 0.00 : ffff800010476f70: ldr w1, [x27, #40] 0.00 : ffff800010476f74: str w1, [x25, #40] : switch (bio_op(bio)) { 0.00 : ffff800010476f78: b.eq ffff800010477190 // b.none 0.00 : ffff800010476f7c: b.ls ffff800010477378 // b.plast 0.00 : ffff800010476f80: cmp w0, #0x7 0.00 : ffff800010476f84: b.ne ffff800010477288 // b.any : bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0]; 0.00 : ffff800010476f88: ldrh w0, [x25, #96] 0.00 : ffff800010476f8c: ldr x2, [x25, #104] 0.00 : ffff800010476f90: ldr x1, [x27, #104] 0.00 : ffff800010476f94: add w3, w0, #0x1 0.00 : ffff800010476f98: ubfiz x0, x0, #4, #16 0.00 : ffff800010476f9c: strh w3, [x25, #96] 0.00 : ffff800010476fa0: add x2, x2, x0 0.00 : ffff800010476fa4: ldp x0, x1, [x1] 0.00 : ffff800010476fa8: stp x0, x1, [x2] 0.00 : ffff800010476fac: ldr w2, [x27, #16] : bio_integrity(): : : #if defined(CONFIG_BLK_DEV_INTEGRITY) : : static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) : { : if (bio->bi_opf & REQ_INTEGRITY) 0.00 : ffff800010476fb0: tbz w2, #16, ffff800010476fd0 : bounce_clone_bio(): : if (bio_integrity(bio_src)) { 0.00 : ffff800010476fb4: ldr x0, [x27, #88] 0.00 : ffff800010476fb8: cbz x0, ffff800010476fd0 : ret = bio_integrity_clone(bio, bio_src, gfp_mask); 0.00 : ffff800010476fbc: mov w2, #0xc00 // #3072 0.00 : ffff800010476fc0: mov x1, x27 0.00 : ffff800010476fc4: mov x0, x25 0.00 : ffff800010476fc8: bl ffff800010480548 : if (ret < 0) { 0.00 : ffff800010476fcc: tbnz w0, #31, ffff8000104773f4 : bio_clone_blkg_association(bio, bio_src); 0.00 : ffff800010476fd0: mov x1, x27 0.00 : ffff800010476fd4: mov x0, x25 0.00 : ffff800010476fd8: bl ffff8000104567f0 : blkcg_bio_issue_init(): : return false; : } : : static inline void blkcg_bio_issue_init(struct bio *bio) : { : bio_issue_init(&bio->bi_issue, bio_sectors(bio)); 0.00 : ffff800010476fdc: ldr w26, [x25, #40] : bio_issue_init(): : : static inline void bio_issue_init(struct bio_issue *issue, : sector_t size) : { : size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1; : issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | 0.00 : ffff800010476fe0: ldr x27, [x25, #80] : ktime_get_ns(): : return ktime_mono_to_any(mono, TK_OFFS_REAL); : } : : static inline u64 ktime_get_ns(void) : { : return ktime_to_ns(ktime_get()); 0.00 : ffff800010476fe4: bl ffff80001016c938 : bio_issue_init(): : (ktime_get_ns() & BIO_ISSUE_TIME_MASK) | 0.00 : ffff800010476fe8: and x0, x0, #0x7ffffffffffff : blkcg_bio_issue_init(): 0.00 : ffff800010476fec: lsr w26, w26, #9 : bio_issue_init(): : issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | 0.00 : ffff800010476ff0: and x27, x27, #0x8000000000000000 : ((u64)size << BIO_ISSUE_SIZE_SHIFT)); 0.00 : ffff800010476ff4: ubfiz x26, x26, #51, #12 : (ktime_get_ns() & BIO_ISSUE_TIME_MASK) | 0.00 : ffff800010476ff8: orr x26, x26, x27 0.00 : ffff800010476ffc: orr x0, x26, x0 : issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | 0.00 : ffff800010477000: str x0, [x25, #80] : __blk_queue_bounce(): : for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) { 0.00 : ffff800010477004: ldrh w4, [x25, #96] : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 0.00 : ffff800010477008: and w23, w23, #0x1 : __blk_queue_bounce(): 0.00 : ffff80001047700c: ldr x26, [x25, #104] 0.00 : ffff800010477010: cbz w4, ffff800010477060 0.00 : ffff800010477014: mov w27, #0x0 // #0 : if (page_to_pfn(page) <= q->limits.bounce_pfn) 0.00 : ffff800010477018: ldr x0, [x24, #1880] : struct page *page = to->bv_page; 0.00 : ffff80001047701c: ldr x28, [x26] : if (page_to_pfn(page) <= q->limits.bounce_pfn) 0.00 : ffff800010477020: ldr x1, [x22, #1040] 0.00 : ffff800010477024: sub x0, x28, x0 0.00 : ffff800010477028: cmp x1, x0, asr #6 0.00 : ffff80001047702c: b.cs ffff800010477050 // b.hs, b.nlast : to->bv_page = mempool_alloc(pool, q->bounce_gfp); 0.00 : ffff800010477030: ldr w1, [x22, #120] 0.00 : ffff800010477034: mov x0, x21 0.00 : ffff800010477038: bl ffff8000101d7640 0.00 : ffff80001047703c: str x0, [x26] : inc_zone_page_state(to->bv_page, NR_BOUNCE); 0.00 : ffff800010477040: mov w1, #0xa // #10 0.00 : ffff800010477044: bl ffff8000101f56f0 : if (rw == WRITE) { 0.00 : ffff800010477048: cbnz w23, ffff8000104770c4 0.00 : ffff80001047704c: ldrh w4, [x25, #96] : for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) { 0.00 : ffff800010477050: add w27, w27, #0x1 0.00 : ffff800010477054: add x26, x26, #0x10 0.00 : ffff800010477058: cmp w27, w4 0.00 : ffff80001047705c: b.cc ffff800010477018 // b.lo, b.ul, b.last : bio->bi_flags |= (1 << BIO_BOUNCED); 0.00 : ffff800010477060: ldrh w0, [x25, #20] : if (pool == &page_pool) { 0.00 : ffff800010477064: add x20, x20, #0xc90 0.00 : ffff800010477068: add x20, x20, #0x48 : bio->bi_flags |= (1 << BIO_BOUNCED); 0.00 : ffff80001047706c: orr w0, w0, #0x4 0.00 : ffff800010477070: strh w0, [x25, #20] : if (pool == &page_pool) { 0.00 : ffff800010477074: cmp x21, x20 0.00 : ffff800010477078: b.eq ffff8000104771a8 // b.none : bio->bi_end_io = bounce_end_io_write_isa; 0.00 : ffff80001047707c: adrp x1, ffff800010476000 0.00 : ffff800010477080: adrp x0, ffff800010476000 0.00 : ffff800010477084: add x1, x1, #0xba8 0.00 : ffff800010477088: add x0, x0, #0x8e0 0.00 : ffff80001047708c: cmp w23, #0x0 0.00 : ffff800010477090: csel x0, x0, x1, ne // ne = any 0.00 : ffff800010477094: str x0, [x25, #56] : bio->bi_private = *bio_orig; 0.00 : ffff800010477098: ldr x0, [x19] 0.00 : ffff80001047709c: str x0, [x25, #64] : *bio_orig = bio; 0.00 : ffff8000104770a0: str x25, [x19] 0.00 : ffff8000104770a4: ldp x19, x20, [x29, #16] 0.00 : ffff8000104770a8: ldp x21, x22, [x29, #32] 0.00 : ffff8000104770ac: ldp x24, x25, [x29, #56] 0.00 : ffff8000104770b0: ldp x26, x27, [x29, #72] 0.00 : ffff8000104770b4: ldr x28, [x29, #88] 0.00 : ffff8000104770b8: b ffff800010476d4c 62.71 : ffff8000104770bc: ldp x20, x21, [x29, #24] 0.00 : ffff8000104770c0: b ffff800010476d4c : flush_dcache_page(page); 0.00 : ffff8000104770c4: mov x0, x28 0.00 : ffff8000104770c8: bl ffff8000100a3110 : lowmem_page_address(): : */ : #include : : static __always_inline void *lowmem_page_address(const struct page *page) : { : return page_to_virt(page); 0.00 : ffff8000104770cc: ldr x0, [x26] 0.00 : ffff8000104770d0: mov x1, #0x200000 // #2097152 0.00 : ffff8000104770d4: movk x1, #0x200, lsl #32 : __blk_queue_bounce(): : vto = page_address(to->bv_page) + to->bv_offset; 0.00 : ffff8000104770d8: ldr w5, [x26, #12] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000104770dc: mrs x4, sp_el0 : lowmem_page_address(): 0.00 : ffff8000104770e0: add x0, x0, x1 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000104770e4: ldr w1, [x4, #16] : lowmem_page_address(): 0.00 : ffff8000104770e8: lsr x0, x0, #6 : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000104770ec: add w1, w1, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000104770f0: str w1, [x4, #16] : lowmem_page_address(): 0.00 : ffff8000104770f4: mov x1, #0xffff000000000000 // #-281474976710656 0.00 : ffff8000104770f8: add x0, x1, x0, lsl #12 : __blk_queue_bounce(): 0.00 : ffff8000104770fc: add x0, x0, x5 : pagefault_disabled_inc(): : } : #endif : : static __always_inline void pagefault_disabled_inc(void) : { : current->pagefault_disabled++; 0.00 : ffff800010477100: ldr w1, [x4, #2448] 0.00 : ffff800010477104: str x4, [x29, #104] 0.00 : ffff800010477108: add w1, w1, #0x1 0.00 : ffff80001047710c: str w1, [x4, #2448] : lowmem_page_address(): 0.00 : ffff800010477110: mov x1, #0x200000 // #2097152 0.00 : ffff800010477114: mov x3, #0xffff000000000000 // #-281474976710656 0.00 : ffff800010477118: movk x1, #0x200, lsl #32 0.00 : ffff80001047711c: add x1, x28, x1 : __blk_queue_bounce(): : vfrom = kmap_atomic(page) + to->bv_offset; 0.00 : ffff800010477120: ldp w2, w5, [x26, #8] : lowmem_page_address(): 0.00 : ffff800010477124: lsr x1, x1, #6 0.00 : ffff800010477128: add x1, x3, x1, lsl #12 : __blk_queue_bounce(): : memcpy(vto, vfrom, to->bv_len); 0.00 : ffff80001047712c: add x1, x1, x5 0.00 : ffff800010477130: bl ffff800010cb7dc0 <__memcpy> : pagefault_disabled_dec(): : } : : static __always_inline void pagefault_disabled_dec(void) : { : current->pagefault_disabled--; 0.00 : ffff800010477134: ldr x4, [x29, #104] 0.00 : ffff800010477138: ldr w0, [x4, #2448] 0.00 : ffff80001047713c: sub w0, w0, #0x1 0.00 : ffff800010477140: str w0, [x4, #2448] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010477144: ldr x0, [x4, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010477148: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001047714c: str w0, [x4, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010477150: cbz x0, ffff80001047715c : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010477154: ldr x0, [x4, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010477158: cbnz x0, ffff80001047704c : __kunmap_atomic(): : #define kmap_atomic_prot(page, prot) kmap_atomic(page) : : static inline void __kunmap_atomic(void *addr) : { : pagefault_enable(); : preempt_enable(); 0.00 : ffff80001047715c: bl ffff800010cd2e50 0.00 : ffff800010477160: ldrh w4, [x25, #96] 0.00 : ffff800010477164: b ffff800010477050 : bvec_iter_advance(): 0.00 : ffff800010477168: adrp x1, ffff800011ad8000 0.00 : ffff80001047716c: ldrb w0, [x1, #1296] 0.00 : ffff800010477170: cbnz w0, ffff800010476fb0 0.00 : ffff800010477174: mov w2, #0x1 // #1 0.00 : ffff800010477178: adrp x0, ffff8000111e9000 0.00 : ffff80001047717c: strb w2, [x1, #1296] 0.00 : ffff800010477180: add x0, x0, #0x680 0.00 : ffff800010477184: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010477188: brk #0x800 0.00 : ffff80001047718c: nop 0.00 : ffff800010477190: ldr w2, [x27, #16] 0.00 : ffff800010477194: b ffff800010476fb0 : bio_segments(): : switch (bio_op(bio)) { 0.00 : ffff800010477198: cmp w0, #0x9 0.00 : ffff80001047719c: b.ne ffff8000104771d0 // b.any : return 0; 0.00 : ffff8000104771a0: mov w1, #0x0 // #0 0.00 : ffff8000104771a4: b ffff800010476f30 : __blk_queue_bounce(): : bio->bi_end_io = bounce_end_io_write; 0.00 : ffff8000104771a8: adrp x1, ffff800010476000 0.00 : ffff8000104771ac: adrp x0, ffff800010476000 0.00 : ffff8000104771b0: add x1, x1, #0xbe8 0.00 : ffff8000104771b4: add x0, x0, #0x900 0.00 : ffff8000104771b8: cmp w23, #0x0 0.00 : ffff8000104771bc: csel x0, x0, x1, ne // ne = any 0.00 : ffff8000104771c0: str x0, [x25, #56] 0.00 : ffff8000104771c4: b ffff800010477098 : bio_segments(): : switch (bio_op(bio)) { 0.00 : ffff8000104771c8: cmp w0, #0x3 0.00 : ffff8000104771cc: b.eq ffff8000104771a0 // b.none : bio_for_each_segment(bv, bio, iter) 0.00 : ffff8000104771d0: ldp w7, w5, [x27, #40] : unsigned segs = 0; 0.00 : ffff8000104771d4: mov w1, #0x0 // #0 : bio_for_each_segment(bv, bio, iter) 0.00 : ffff8000104771d8: ldr w3, [x27, #48] 0.00 : ffff8000104771dc: mov w10, #0x1000 // #4096 : bio_no_advance_iter(): : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff8000104771e0: mov w9, #0xfb // #251 0.00 : ffff8000104771e4: nop : bio_segments(): : bio_for_each_segment(bv, bio, iter) 0.00 : ffff8000104771e8: cbz w7, ffff800010476f30 0.00 : ffff8000104771ec: ldr x8, [x27, #104] 0.00 : ffff8000104771f0: ubfiz x0, x5, #4, #32 : segs++; 0.00 : ffff8000104771f4: add w1, w1, #0x1 : bio_for_each_segment(bv, bio, iter) 0.00 : ffff8000104771f8: add x0, x8, x0 0.00 : ffff8000104771fc: ldp w4, w0, [x0, #8] 0.00 : ffff800010477200: sub w6, w4, w3 0.00 : ffff800010477204: add w0, w3, w0 0.00 : ffff800010477208: cmp w6, w7 0.00 : ffff80001047720c: and w0, w0, #0xfff 0.00 : ffff800010477210: csel w6, w6, w7, ls // ls = plast 0.00 : ffff800010477214: sub w0, w10, w0 0.00 : ffff800010477218: cmp w0, w6 0.00 : ffff80001047721c: csel w0, w0, w6, ls // ls = plast : bio_advance_iter(): : if (bio_no_advance_iter(bio)) 0.00 : ffff800010477220: ldr w6, [x27, #16] : bio_no_advance_iter(): : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff800010477224: and w11, w6, #0xff 0.00 : ffff800010477228: and w6, w6, w9 0.00 : ffff80001047722c: sub w11, w11, #0x5 : bio_op(bio) == REQ_OP_WRITE_SAME || 0.00 : ffff800010477230: tst w11, #0xfffffffb 0.00 : ffff800010477234: ccmp w6, #0x3, #0x4, ne // ne = any 0.00 : ffff800010477238: b.ne ffff800010477244 // b.any : bio_advance_iter(): : iter->bi_size -= bytes; 0.00 : ffff80001047723c: sub w7, w7, w0 0.00 : ffff800010477240: b ffff8000104771e8 : bvec_iter_advance(): 0.00 : ffff800010477244: cmp w0, w7 0.00 : ffff800010477248: b.hi ffff8000104773bc // b.pmore : iter->bi_size -= bytes; 0.00 : ffff80001047724c: sub w7, w7, w0 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010477250: adds w3, w3, w0 0.00 : ffff800010477254: b.eq ffff8000104771e8 // b.none 0.00 : ffff800010477258: cmp w4, w3 0.00 : ffff80001047725c: b.ls ffff800010477278 // b.plast 0.00 : ffff800010477260: b ffff8000104771e8 0.00 : ffff800010477264: ubfiz x0, x5, #4, #32 0.00 : ffff800010477268: add x0, x8, x0 0.00 : ffff80001047726c: ldr w4, [x0, #8] 0.00 : ffff800010477270: cmp w4, w3 0.00 : ffff800010477274: b.hi ffff8000104771e8 // b.pmore : idx++; 0.00 : ffff800010477278: add w5, w5, #0x1 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff80001047727c: subs w3, w3, w4 0.00 : ffff800010477280: b.ne ffff800010477264 // b.any 0.00 : ffff800010477284: b ffff8000104771e8 : bounce_clone_bio(): : switch (bio_op(bio)) { 0.00 : ffff800010477288: cmp w0, #0x9 0.00 : ffff80001047728c: b.eq ffff800010477190 // b.none : bio_for_each_segment(bv, bio_src, iter) 0.00 : ffff800010477290: ldp w5, w3, [x27, #40] 0.00 : ffff800010477294: mov w8, #0x1000 // #4096 0.00 : ffff800010477298: ldr w1, [x27, #48] : bio_no_advance_iter(): : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff80001047729c: mov w7, #0xfb // #251 : bounce_clone_bio(): 0.00 : ffff8000104772a0: cbz w5, ffff800010477190 0.00 : ffff8000104772a4: ldr x2, [x27, #104] 0.00 : ffff8000104772a8: ubfiz x6, x3, #4, #32 : bio->bi_io_vec[bio->bi_vcnt++] = bv; 0.00 : ffff8000104772ac: ldr x11, [x25, #104] : bio_for_each_segment(bv, bio_src, iter) 0.00 : ffff8000104772b0: add x0, x2, x6 0.00 : ffff8000104772b4: ldr x9, [x2, x6] 0.00 : ffff8000104772b8: ldr w2, [x0, #12] 0.00 : ffff8000104772bc: ldr w0, [x0, #8] 0.00 : ffff8000104772c0: add w2, w1, w2 0.00 : ffff8000104772c4: sub w0, w0, w1 0.00 : ffff8000104772c8: and w10, w2, #0xfff 0.00 : ffff8000104772cc: cmp w0, w5 0.00 : ffff8000104772d0: sub w4, w8, w10 0.00 : ffff8000104772d4: csel w0, w0, w5, ls // ls = plast 0.00 : ffff8000104772d8: lsr w2, w2, #12 0.00 : ffff8000104772dc: cmp w0, w4 0.00 : ffff8000104772e0: csel w0, w0, w4, ls // ls = plast : bio->bi_io_vec[bio->bi_vcnt++] = bv; 0.00 : ffff8000104772e4: ldrh w4, [x25, #96] : bio_for_each_segment(bv, bio_src, iter) 0.00 : ffff8000104772e8: add x2, x9, x2, lsl #6 : bio->bi_io_vec[bio->bi_vcnt++] = bv; 0.00 : ffff8000104772ec: add w9, w4, #0x1 0.00 : ffff8000104772f0: strh w9, [x25, #96] 0.00 : ffff8000104772f4: ubfiz x4, x4, #4, #16 0.00 : ffff8000104772f8: add x9, x11, x4 0.00 : ffff8000104772fc: str x2, [x11, x4] 0.00 : ffff800010477300: stp w0, w10, [x9, #8] : bio_advance_iter(): : if (bio_no_advance_iter(bio)) 0.00 : ffff800010477304: ldr w2, [x27, #16] : bio_no_advance_iter(): : return bio_op(bio) == REQ_OP_DISCARD || 0.00 : ffff800010477308: and w4, w2, #0xff 0.00 : ffff80001047730c: and w9, w2, w7 0.00 : ffff800010477310: sub w4, w4, #0x5 : bio_op(bio) == REQ_OP_WRITE_SAME || 0.00 : ffff800010477314: tst w4, #0xfffffffb 0.00 : ffff800010477318: ccmp w9, #0x3, #0x4, ne // ne = any 0.00 : ffff80001047731c: b.ne ffff800010477328 // b.any : bio_advance_iter(): : iter->bi_size -= bytes; 0.00 : ffff800010477320: sub w5, w5, w0 0.00 : ffff800010477324: b ffff8000104772a0 : bvec_iter_advance(): : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff800010477328: cmp w0, w5 : bio_advance_iter(): : bvec_iter_advance(bio->bi_io_vec, iter, bytes); 0.00 : ffff80001047732c: ldr x4, [x27, #104] : bvec_iter_advance(): 0.00 : ffff800010477330: b.hi ffff800010477168 // b.pmore : iter->bi_size -= bytes; 0.00 : ffff800010477334: sub w5, w5, w0 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010477338: adds w1, w1, w0 0.00 : ffff80001047733c: b.eq ffff8000104772a0 // b.none 0.00 : ffff800010477340: add x6, x4, x6 0.00 : ffff800010477344: ldr w0, [x6, #8] 0.00 : ffff800010477348: cmp w1, w0 0.00 : ffff80001047734c: b.cs ffff800010477368 // b.hs, b.nlast 0.00 : ffff800010477350: b ffff8000104772a0 0.00 : ffff800010477354: ubfiz x0, x3, #4, #32 0.00 : ffff800010477358: add x0, x4, x0 0.00 : ffff80001047735c: ldr w0, [x0, #8] 0.00 : ffff800010477360: cmp w0, w1 0.00 : ffff800010477364: b.hi ffff8000104772a0 // b.pmore : idx++; 0.00 : ffff800010477368: add w3, w3, #0x1 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff80001047736c: subs w1, w1, w0 0.00 : ffff800010477370: b.ne ffff800010477354 // b.any 0.00 : ffff800010477374: b ffff8000104772a0 : bounce_clone_bio(): : switch (bio_op(bio)) { 0.00 : ffff800010477378: cmp w0, #0x3 0.00 : ffff80001047737c: b.ne ffff800010477290 // b.any 0.00 : ffff800010477380: ldr w2, [x27, #16] 0.00 : ffff800010477384: b ffff800010476fb0 : __blk_queue_bounce(): : bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split); 0.00 : ffff800010477388: add x3, x20, #0xc90 0.00 : ffff80001047738c: mov w2, #0xc00 // #3072 0.00 : ffff800010477390: add x3, x3, #0x208 0.00 : ffff800010477394: mov x0, x27 0.00 : ffff800010477398: bl ffff800010457260 0.00 : ffff80001047739c: mov x27, x0 : bio_chain(bio, *bio_orig); 0.00 : ffff8000104773a0: ldr x1, [x19] 0.00 : ffff8000104773a4: bl ffff800010455df8 : generic_make_request(*bio_orig); 0.00 : ffff8000104773a8: ldr x0, [x19] 0.00 : ffff8000104773ac: bl ffff80001045b4c0 : *bio_orig = bio; 0.00 : ffff8000104773b0: str x27, [x19] 0.00 : ffff8000104773b4: b ffff800010476f0c : blk_queue_bounce(): : BUG_ON(!mempool_initialized(&isa_page_pool)); 0.00 : ffff8000104773b8: brk #0x800 : bvec_iter_advance(): : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff8000104773bc: adrp x3, ffff800011ad8000 0.00 : ffff8000104773c0: ldrb w0, [x3, #1296] 0.00 : ffff8000104773c4: cbnz w0, ffff800010476f30 0.00 : ffff8000104773c8: mov w4, #0x1 // #1 0.00 : ffff8000104773cc: str w1, [x29, #100] 0.00 : ffff8000104773d0: strb w4, [x3, #1296] 0.00 : ffff8000104773d4: adrp x0, ffff8000111e9000 0.00 : ffff8000104773d8: str x2, [x29, #104] 0.00 : ffff8000104773dc: add x0, x0, #0x680 0.00 : ffff8000104773e0: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff8000104773e4: brk #0x800 0.00 : ffff8000104773e8: ldr w1, [x29, #100] 0.00 : ffff8000104773ec: ldr x2, [x29, #104] 0.00 : ffff8000104773f0: b ffff800010476f30 : bounce_clone_bio(): : bio_put(bio); 0.00 : ffff8000104773f4: mov x0, x25 : return NULL; 0.00 : ffff8000104773f8: mov x25, #0x0 // #0 : bio_put(bio); 0.00 : ffff8000104773fc: bl ffff800010456ab8 0.00 : ffff800010477400: b ffff800010477004 Percent | Source code & Disassembly of vmlinux for cycles (14 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b710 : : tick_sched_do_timer(): : write_sequnlock(&jiffies_lock); : return period; : } : : static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) : { 0.00 : ffff80001017b710: stp x29, x30, [sp, #-32]! : * jiffies_lock. : * : * If nohz_full is enabled, this should not happen because the : * tick_do_timer_cpu never relinquishes. : */ : if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) { 0.00 : ffff80001017b714: adrp x4, ffff80001190a000 <__per_cpu_offset+0x6d8> : int cpu = smp_processor_id(); 0.00 : ffff80001017b718: adrp x2, ffff80001151d000 0.00 : ffff80001017b71c: add x2, x2, #0x18 : { 6.86 : ffff80001017b720: mov x29, sp 0.00 : ffff80001017b724: str x19, [sp, #16] 0.00 : ffff80001017b728: mov x19, x0 : if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) { 0.00 : ffff80001017b72c: ldr w3, [x4, #712] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001017b730: mrs x5, tpidr_el1 : tick_sched_do_timer(): 0.00 : ffff80001017b734: cmn w3, #0x1 : int cpu = smp_processor_id(); 0.00 : ffff80001017b738: ldr w2, [x2, x5] : if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) { 0.00 : ffff80001017b73c: b.eq ffff80001017b764 // b.none : tick_do_timer_cpu = cpu; : } : #endif : : /* Check, if the jiffies need an update */ : if (tick_do_timer_cpu == cpu) 7.21 : ffff80001017b740: cmp w3, w2 0.00 : ffff80001017b744: b.eq ffff80001017b768 // b.none : tick_do_update_jiffies64(now); : : if (ts->inidle) 35.69 : ffff80001017b748: ldrb w0, [x19, #76] 0.00 : ffff80001017b74c: tbz w0, #0, ffff80001017b758 : ts->got_idle_tick = 1; 0.00 : ffff80001017b750: orr w0, w0, #0x10 0.00 : ffff80001017b754: strb w0, [x19, #76] : } 21.36 : ffff80001017b758: ldr x19, [sp, #16] 0.00 : ffff80001017b75c: ldp x29, x30, [sp], #32 0.00 : ffff80001017b760: ret : tick_do_timer_cpu = cpu; 0.00 : ffff80001017b764: str w2, [x4, #712] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001017b768: adrp x0, ffff800011b22000 : tick_do_update_jiffies64(): : if (delta < tick_period) 0.00 : ffff80001017b76c: adrp x2, ffff800011b22000 : __read_once_size(): 0.00 : ffff80001017b770: ldr x0, [x0, #1328] : tick_do_update_jiffies64(): 28.88 : ffff80001017b774: ldr x2, [x2, #968] : delta = ktime_sub(now, READ_ONCE(last_jiffies_update)); 0.00 : ffff80001017b778: sub x0, x1, x0 : if (delta < tick_period) 0.00 : ffff80001017b77c: cmp x0, x2 0.00 : ffff80001017b780: b.lt ffff80001017b748 // b.tstop 0.00 : ffff80001017b784: mov x0, x1 0.00 : ffff80001017b788: bl ffff80001017b488 0.00 : ffff80001017b78c: b ffff80001017b748 Percent | Source code & Disassembly of vmlinux for cycles (49 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010456970 : : bio_associate_blkg(): : * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is : * already associated, the css is reused and association redone as the : * request_queue may have changed. : */ : void bio_associate_blkg(struct bio *bio) : { 10.32 : ffff800010456970: stp x29, x30, [sp, #-32]! 0.00 : ffff800010456974: mov x29, sp 4.08 : ffff800010456978: str x19, [sp, #16] 0.00 : ffff80001045697c: mov x19, x0 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 2.06 : ffff800010456980: bl ffff80001015c1b0 <__rcu_read_lock> : bio_associate_blkg(): : struct cgroup_subsys_state *css; : : rcu_read_lock(); : : if (bio->bi_blkg) 0.00 : ffff800010456984: ldr x0, [x19, #72] 0.00 : ffff800010456988: cbz x0, ffff8000104569b0 : bio_blkcg(): : */ : static inline struct blkcg *bio_blkcg(struct bio *bio) : { : if (bio && bio->bi_blkg) : return bio->bi_blkg->blkcg; : return NULL; 0.00 : ffff80001045698c: mov x1, #0x0 // #0 : if (bio && bio->bi_blkg) 0.00 : ffff800010456990: cbz x19, ffff800010456998 : return bio->bi_blkg->blkcg; 0.00 : ffff800010456994: ldr x1, [x0, #40] : bio_associate_blkg(): : css = &bio_blkcg(bio)->css; : else : css = blkcg_css(); : : bio_associate_blkg_from_css(bio, css); 46.68 : ffff800010456998: mov x0, x19 0.00 : ffff80001045699c: bl ffff8000104568f0 : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 2.06 : ffff8000104569a0: bl ffff800010160d20 <__rcu_read_unlock> : bio_associate_blkg(): : : rcu_read_unlock(); : } 2.10 : ffff8000104569a4: ldr x19, [sp, #16] 0.00 : ffff8000104569a8: ldp x29, x30, [sp], #32 0.00 : ffff8000104569ac: ret : blkcg_css(): : css = kthread_blkcg(); 4.16 : ffff8000104569b0: bl ffff80001010bce8 5.92 : ffff8000104569b4: mov x1, x0 : if (css) 0.00 : ffff8000104569b8: cbnz x0, ffff800010456998 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000104569bc: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000104569c0: ldr x0, [x0, #2000] : task_css(): : * See task_css_check(). : */ : static inline struct cgroup_subsys_state *task_css(struct task_struct *task, : int subsys_id) : { : return task_css_check(task, subsys_id, false); 22.61 : ffff8000104569c4: ldr x1, [x0, #24] 0.00 : ffff8000104569c8: b ffff800010456998 Percent | Source code & Disassembly of vmlinux for cycles (34 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102cd4c8 <__fsnotify_parent>: : __fsnotify_parent(): : spin_unlock(&inode->i_lock); : } : : /* Notify this dentry's parent about a child's events. */ : int __fsnotify_parent(const struct path *path, struct dentry *dentry, __u32 mask) : { 0.00 : ffff8000102cd4c8: stp x29, x30, [sp, #-144]! 0.00 : ffff8000102cd4cc: mov x29, sp 5.65 : ffff8000102cd4d0: stp x19, x20, [sp, #16] 0.00 : ffff8000102cd4d4: adrp x19, ffff800011909000 0.00 : ffff8000102cd4d8: stp x21, x22, [sp, #32] 0.00 : ffff8000102cd4dc: add x3, x19, #0x908 11.90 : ffff8000102cd4e0: str x23, [sp, #48] 0.00 : ffff8000102cd4e4: mov w22, w2 0.00 : ffff8000102cd4e8: mov x23, x0 0.00 : ffff8000102cd4ec: mov x20, x1 2.97 : ffff8000102cd4f0: ldr x4, [x3] 0.00 : ffff8000102cd4f4: str x4, [x29, #136] 0.00 : ffff8000102cd4f8: mov x4, #0x0 // #0 : struct dentry *parent; : struct inode *p_inode; : int ret = 0; : : if (!dentry) 0.00 : ffff8000102cd4fc: cbz x1, ffff8000102cd584 <__fsnotify_parent+0xbc> : dentry = path->dentry; : : if (!(dentry->d_flags & DCACHE_FSNOTIFY_PARENT_WATCHED)) 2.94 : ffff8000102cd500: ldr w0, [x20] : return 0; 0.00 : ffff8000102cd504: mov w21, #0x0 // #0 : if (!(dentry->d_flags & DCACHE_FSNOTIFY_PARENT_WATCHED)) 0.00 : ffff8000102cd508: tbnz w0, #14, ffff8000102cd538 <__fsnotify_parent+0x70> : } : : dput(parent); : : return ret; : } 53.18 : ffff8000102cd50c: add x19, x19, #0x908 0.00 : ffff8000102cd510: mov w0, w21 5.87 : ffff8000102cd514: ldr x2, [x29, #136] 0.00 : ffff8000102cd518: ldr x1, [x19] 0.00 : ffff8000102cd51c: eor x1, x2, x1 0.00 : ffff8000102cd520: cbnz x1, ffff8000102cd610 <__fsnotify_parent+0x148> 0.00 : ffff8000102cd524: ldp x19, x20, [sp, #16] 8.90 : ffff8000102cd528: ldp x21, x22, [sp, #32] 8.59 : ffff8000102cd52c: ldr x23, [sp, #48] 0.00 : ffff8000102cd530: ldp x29, x30, [sp], #144 0.00 : ffff8000102cd534: ret 0.00 : ffff8000102cd538: stp x24, x25, [x29, #56] : parent = dget_parent(dentry); 0.00 : ffff8000102cd53c: mov x0, x20 0.00 : ffff8000102cd540: bl ffff80001029afe8 : p_inode = parent->d_inode; 0.00 : ffff8000102cd544: ldr x25, [x0, #48] : parent = dget_parent(dentry); 0.00 : ffff8000102cd548: mov x24, x0 : if (unlikely(!fsnotify_inode_watches_children(p_inode))) { 0.00 : ffff8000102cd54c: ldr w1, [x25, #556] : fsnotify_inode_watches_children(): : extern u32 fsnotify_get_cookie(void); : : static inline int fsnotify_inode_watches_children(struct inode *inode) : { : /* FS_EVENT_ON_CHILD is set if the inode may care */ : if (!(inode->i_fsnotify_mask & FS_EVENT_ON_CHILD)) 0.00 : ffff8000102cd550: tbz w1, #27, ffff8000102cd5cc <__fsnotify_parent+0x104> : return 0; : /* this inode might care about child events, does it care about the : * specific set of events that can happen on a child? */ : return inode->i_fsnotify_mask & FS_EVENTS_POSS_ON_CHILD; 0.00 : ffff8000102cd554: mov w0, #0x103f // #4159 0.00 : ffff8000102cd558: movk w0, #0x7, lsl #16 : __fsnotify_parent(): 0.00 : ffff8000102cd55c: tst w1, w0 0.00 : ffff8000102cd560: b.eq ffff8000102cd5cc <__fsnotify_parent+0x104> // b.none : } else if (p_inode->i_fsnotify_mask & mask & ALL_FSNOTIFY_EVENTS) { 0.00 : ffff8000102cd564: mov w0, #0x1007ffff // #268959743 0.00 : ffff8000102cd568: and w0, w22, w0 0.00 : ffff8000102cd56c: tst w0, w1 0.00 : ffff8000102cd570: b.ne ffff8000102cd58c <__fsnotify_parent+0xc4> // b.any : dput(parent); 0.00 : ffff8000102cd574: mov x0, x24 0.00 : ffff8000102cd578: bl ffff80001029ac00 0.00 : ffff8000102cd57c: ldp x24, x25, [x29, #56] : return ret; 0.00 : ffff8000102cd580: b ffff8000102cd50c <__fsnotify_parent+0x44> : dentry = path->dentry; 0.00 : ffff8000102cd584: ldr x20, [x0, #8] 0.00 : ffff8000102cd588: b ffff8000102cd500 <__fsnotify_parent+0x38> : mask |= FS_EVENT_ON_CHILD; 0.00 : ffff8000102cd58c: orr w22, w22, #0x8000000 : take_dentry_name_snapshot(&name, dentry); 0.00 : ffff8000102cd590: mov x1, x20 0.00 : ffff8000102cd594: add x0, x29, #0x58 0.00 : ffff8000102cd598: bl ffff8000102989c8 : if (path) 0.00 : ffff8000102cd59c: cbz x23, ffff8000102cd5ec <__fsnotify_parent+0x124> : ret = fsnotify(p_inode, mask, path, FSNOTIFY_EVENT_PATH, 0.00 : ffff8000102cd5a0: mov w5, #0x0 // #0 0.00 : ffff8000102cd5a4: add x4, x29, #0x58 0.00 : ffff8000102cd5a8: mov w3, #0x1 // #1 0.00 : ffff8000102cd5ac: mov x2, x23 0.00 : ffff8000102cd5b0: mov w1, w22 0.00 : ffff8000102cd5b4: mov x0, x25 0.00 : ffff8000102cd5b8: bl ffff8000102cd188 0.00 : ffff8000102cd5bc: mov w21, w0 : release_dentry_name_snapshot(&name); 0.00 : ffff8000102cd5c0: add x0, x29, #0x58 0.00 : ffff8000102cd5c4: bl ffff800010298a58 0.00 : ffff8000102cd5c8: b ffff8000102cd574 <__fsnotify_parent+0xac> : __fsnotify_update_child_dentry_flags(): : if (!S_ISDIR(inode->i_mode)) 0.00 : ffff8000102cd5cc: ldrh w0, [x25] : __fsnotify_parent(): : int ret = 0; 0.00 : ffff8000102cd5d0: mov w21, #0x0 // #0 : __fsnotify_update_child_dentry_flags(): : if (!S_ISDIR(inode->i_mode)) 0.00 : ffff8000102cd5d4: and w0, w0, #0xf000 0.00 : ffff8000102cd5d8: cmp w0, #0x4, lsl #12 0.00 : ffff8000102cd5dc: b.ne ffff8000102cd574 <__fsnotify_parent+0xac> // b.any 0.00 : ffff8000102cd5e0: mov x0, x25 0.00 : ffff8000102cd5e4: bl ffff8000102cd088 <__fsnotify_update_child_dentry_flags.part.5> 0.00 : ffff8000102cd5e8: b ffff8000102cd574 <__fsnotify_parent+0xac> : __fsnotify_parent(): : ret = fsnotify(p_inode, mask, dentry->d_inode, FSNOTIFY_EVENT_INODE, 0.00 : ffff8000102cd5ec: ldr x2, [x20, #48] 0.00 : ffff8000102cd5f0: mov w5, #0x0 // #0 0.00 : ffff8000102cd5f4: add x4, x29, #0x58 0.00 : ffff8000102cd5f8: mov w3, #0x2 // #2 0.00 : ffff8000102cd5fc: mov w1, w22 0.00 : ffff8000102cd600: mov x0, x25 0.00 : ffff8000102cd604: bl ffff8000102cd188 0.00 : ffff8000102cd608: mov w21, w0 0.00 : ffff8000102cd60c: b ffff8000102cd5c0 <__fsnotify_parent+0xf8> 0.00 : ffff8000102cd610: stp x24, x25, [x29, #56] : } 0.00 : ffff8000102cd614: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (36 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104647c0 : : list_add_tail(): : * Insert a new entry before the specified head. : * This is useful for implementing queues. : */ : static inline void list_add_tail(struct list_head *new, struct list_head *head) : { : __list_add(new, head->prev, head); 0.00 : ffff8000104647c0: ldr x2, [x0, #8] : blk_add_rq_to_plug(): : hctx->queue->mq_ops->commit_rqs(hctx); : } : : static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) : { : list_add_tail(&rq->queuelist, &plug->mq_list); 0.00 : ffff8000104647c4: add x3, x1, #0x48 : __list_add(): : next->prev = new; 0.00 : ffff8000104647c8: str x3, [x0, #8] : new->prev = prev; 0.00 : ffff8000104647cc: stp x0, x2, [x1, #72] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 2.80 : ffff8000104647d0: str x3, [x2] : blk_add_rq_to_plug(): : plug->rq_count++; 0.00 : ffff8000104647d4: ldrh w2, [x0, #32] : if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) { 0.00 : ffff8000104647d8: ldrb w3, [x0, #34] : plug->rq_count++; 0.00 : ffff8000104647dc: add w2, w2, #0x1 0.00 : ffff8000104647e0: strh w2, [x0, #32] : if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) { 0.00 : ffff8000104647e4: cbnz w3, ffff80001046481c : __read_once_size(): : __READ_ONCE_SIZE; 30.32 : ffff8000104647e8: ldr x3, [x0] 50.31 : ffff8000104647ec: ldr x2, [x0] : list_is_singular(): : * list_is_singular - tests whether a list has just one entry. : * @head: the list to test. : */ : static inline int list_is_singular(const struct list_head *head) : { : return !list_empty(head) && (head->next == head->prev); 0.00 : ffff8000104647f0: cmp x0, x3 0.00 : ffff8000104647f4: b.eq ffff800010464804 // b.none 0.00 : ffff8000104647f8: ldr x3, [x0, #8] 0.00 : ffff8000104647fc: cmp x2, x3 8.13 : ffff800010464800: b.eq ffff80001046481c // b.none : blk_add_rq_to_plug(): : struct request *tmp; : : tmp = list_first_entry(&plug->mq_list, struct request, : queuelist); : if (tmp->q != rq->q) 0.00 : ffff800010464804: ldr x1, [x1] 0.00 : ffff800010464808: ldur x2, [x2, #-72] 0.00 : ffff80001046480c: cmp x2, x1 0.00 : ffff800010464810: b.eq ffff80001046481c // b.none : plug->multiple_queues = true; 0.00 : ffff800010464814: mov w1, #0x1 // #1 0.00 : ffff800010464818: strb w1, [x0, #34] : } : } 8.45 : ffff80001046481c: ret Percent | Source code & Disassembly of vmlinux for cycles (34 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010464450 : : blk_mq_start_request(): : * Function used by device drivers to notify the block layer that a request : * is going to be processed now, so blk layer can do proper initializations : * such as starting the timeout timer. : */ : void blk_mq_start_request(struct request *rq) : { 8.80 : ffff800010464450: stp x29, x30, [sp, #-32]! 0.00 : ffff800010464454: mov x29, sp 26.29 : ffff800010464458: stp x19, x20, [sp, #16] 0.00 : ffff80001046445c: mov x19, x0 : struct request_queue *q = rq->q; 2.98 : ffff800010464460: ldr x20, [x0] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010464464: ldr x0, [x20, #104] : blk_mq_start_request(): : : trace_block_rq_issue(q, rq); : : if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 0.00 : ffff800010464468: tst w0, #0x100000 0.00 : ffff80001046446c: b.ne ffff8000104644d8 // b.any : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 26.71 : ffff800010464470: ldr w0, [x19, #208] : blk_mq_start_request(): : rq->stats_sectors = blk_rq_sectors(rq); : rq->rq_flags |= RQF_STATS; : rq_qos_issue(q, rq); : } : : WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 0.00 : ffff800010464474: cbnz w0, ffff800010464510 : : blk_add_timer(rq); 11.82 : ffff800010464478: mov x0, x19 0.00 : ffff80001046447c: bl ffff8000104632d0 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010464480: mov w0, #0x1 // #1 0.00 : ffff800010464484: str w0, [x19, #208] : blk_mq_start_request(): : WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT); : : if (q->dma_drain_size && blk_rq_bytes(rq)) { 8.53 : ffff800010464488: ldr w0, [x20, #240] 0.00 : ffff80001046448c: cbz w0, ffff800010464498 0.00 : ffff800010464490: ldr w0, [x19, #40] 0.00 : ffff800010464494: cbnz w0, ffff8000104644c8 : */ : rq->nr_phys_segments++; : } : : #ifdef CONFIG_BLK_DEV_INTEGRITY : if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE) 2.96 : ffff800010464498: ldr w0, [x19, #24] 0.00 : ffff80001046449c: tbz w0, #16, ffff8000104644bc 0.00 : ffff8000104644a0: and w0, w0, #0xff 0.00 : ffff8000104644a4: cmp w0, #0x1 0.00 : ffff8000104644a8: b.ne ffff8000104644bc // b.any : q->integrity.profile->prepare_fn(rq); 0.00 : ffff8000104644ac: ldr x1, [x20, #200] 0.00 : ffff8000104644b0: mov x0, x19 0.00 : ffff8000104644b4: ldr x1, [x1, #16] 0.00 : ffff8000104644b8: blr x1 : #endif : } 2.95 : ffff8000104644bc: ldp x19, x20, [sp, #16] 8.97 : ffff8000104644c0: ldp x29, x30, [sp], #32 0.00 : ffff8000104644c4: ret : rq->nr_phys_segments++; 0.00 : ffff8000104644c8: ldrh w0, [x19, #194] 0.00 : ffff8000104644cc: add w0, w0, #0x1 0.00 : ffff8000104644d0: strh w0, [x19, #194] 0.00 : ffff8000104644d4: b ffff800010464498 : ktime_get_ns(): : return ktime_mono_to_any(mono, TK_OFFS_REAL); : } : : static inline u64 ktime_get_ns(void) : { : return ktime_to_ns(ktime_get()); 0.00 : ffff8000104644d8: bl ffff80001016c938 : blk_mq_start_request(): : rq->io_start_time_ns = ktime_get_ns(); 0.00 : ffff8000104644dc: str x0, [x19, #184] : blk_rq_sectors(): : : extern unsigned int blk_rq_err_bytes(const struct request *rq); : : static inline unsigned int blk_rq_sectors(const struct request *rq) : { : return blk_rq_bytes(rq) >> SECTOR_SHIFT; 0.00 : ffff8000104644e0: ldr w1, [x19, #40] : blk_mq_start_request(): : rq->rq_flags |= RQF_STATS; 0.00 : ffff8000104644e4: ldr w2, [x19, #28] : blk_rq_sectors(): 0.00 : ffff8000104644e8: lsr w1, w1, #9 : blk_mq_start_request(): 0.00 : ffff8000104644ec: orr w2, w2, #0x20000 0.00 : ffff8000104644f0: str w2, [x19, #28] : rq->stats_sectors = blk_rq_sectors(rq); 0.00 : ffff8000104644f4: strh w1, [x19, #192] : rq_qos_issue(q, rq); 0.00 : ffff8000104644f8: ldr x0, [x20, #24] : rq_qos_issue(): : __rq_qos_done(q->rq_qos, rq); : } : : static inline void rq_qos_issue(struct request_queue *q, struct request *rq) : { : if (q->rq_qos) 0.00 : ffff8000104644fc: cbz x0, ffff800010464470 : __rq_qos_issue(q->rq_qos, rq); 0.00 : ffff800010464500: mov x1, x19 0.00 : ffff800010464504: bl ffff8000104761b8 <__rq_qos_issue> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010464508: ldr w0, [x19, #208] : blk_mq_start_request(): : WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 0.00 : ffff80001046450c: cbz w0, ffff800010464478 0.00 : ffff800010464510: brk #0x800 0.00 : ffff800010464514: b ffff800010464478 Percent | Source code & Disassembly of vmlinux for cycles (35 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010468000 : : blk_mq_try_issue_list_directly(): : return ret; : } : : void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, : struct list_head *list) : { 5.72 : ffff800010468000: stp x29, x30, [sp, #-48]! 0.00 : ffff800010468004: mov x29, sp 5.79 : ffff800010468008: stp x19, x20, [sp, #16] 0.00 : ffff80001046800c: mov x19, x1 0.00 : ffff800010468010: str x21, [sp, #32] 0.00 : ffff800010468014: mov x21, x0 : while (!list_empty(list)) { 0.00 : ffff800010468018: b ffff800010468054 : blk_status_t ret; : struct request *rq = list_first_entry(list, struct request, 5.73 : ffff80001046801c: ldr x1, [x19] 0.00 : ffff800010468020: sub x20, x1, #0x48 : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff800010468024: ldp x3, x2, [x1] : __list_del(): : next->prev = prev; 11.46 : ffff800010468028: str x2, [x3, #8] : blk_mq_try_issue_list_directly(): : queuelist); : : list_del_init(&rq->queuelist); : ret = blk_mq_request_issue_directly(rq, list_empty(list)); 0.00 : ffff80001046802c: mov x0, x20 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 17.08 : ffff800010468030: str x3, [x2] 0.00 : ffff800010468034: str x1, [x1] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010468038: str x1, [x1, #8] : __read_once_size(): : __READ_ONCE_SIZE; 37.45 : ffff80001046803c: ldr x1, [x19] : list_empty(): : * list_empty - tests whether a list is empty : * @head: the list to test. : */ : static inline int list_empty(const struct list_head *head) : { : return READ_ONCE(head->next) == head; 0.00 : ffff800010468040: cmp x19, x1 : blk_mq_try_issue_list_directly(): 0.00 : ffff800010468044: cset w1, eq // eq = none 0.00 : ffff800010468048: bl ffff800010467f70 : if (ret != BLK_STS_OK) { 2.90 : ffff80001046804c: ands w0, w0, #0xff 0.00 : ffff800010468050: b.ne ffff800010468094 // b.any : __read_once_size(): 8.09 : ffff800010468054: ldr x0, [x19] : blk_mq_try_issue_list_directly(): : while (!list_empty(list)) { 0.00 : ffff800010468058: cmp x19, x0 0.00 : ffff80001046805c: b.ne ffff80001046801c // b.any : __read_once_size(): 0.00 : ffff800010468060: ldr x0, [x19] : blk_mq_try_issue_list_directly(): : /* : * If we didn't flush the entire list, we could have told : * the driver there was more coming, but that turned out to : * be a lie. : */ : if (!list_empty(list) && hctx->queue->mq_ops->commit_rqs) 0.00 : ffff800010468064: cmp x19, x0 0.00 : ffff800010468068: b.eq ffff800010468084 // b.none 0.00 : ffff80001046806c: ldr x0, [x21, #208] 0.00 : ffff800010468070: ldr x0, [x0, #48] 0.00 : ffff800010468074: ldr x1, [x0, #8] 0.00 : ffff800010468078: cbz x1, ffff800010468084 : hctx->queue->mq_ops->commit_rqs(hctx); 0.00 : ffff80001046807c: mov x0, x21 0.00 : ffff800010468080: blr x1 : } 5.77 : ffff800010468084: ldp x19, x20, [sp, #16] 0.00 : ffff800010468088: ldr x21, [sp, #32] 0.00 : ffff80001046808c: ldp x29, x30, [sp], #48 0.00 : ffff800010468090: ret : if (ret == BLK_STS_RESOURCE || 0.00 : ffff800010468094: and w2, w0, #0xfffffffb : blk_mq_end_request(rq, ret); 0.00 : ffff800010468098: mov w1, w0 : if (ret == BLK_STS_RESOURCE || 0.00 : ffff80001046809c: cmp w2, #0x9 0.00 : ffff8000104680a0: b.eq ffff8000104680b0 // b.none : blk_mq_end_request(rq, ret); 0.00 : ffff8000104680a4: mov x0, x20 0.00 : ffff8000104680a8: bl ffff800010465d98 0.00 : ffff8000104680ac: b ffff800010468054 : __read_once_size(): 0.00 : ffff8000104680b0: ldr x2, [x19] : blk_mq_try_issue_list_directly(): : blk_mq_request_bypass_insert(rq, false, 0.00 : ffff8000104680b4: mov w1, #0x0 // #0 0.00 : ffff8000104680b8: mov x0, x20 : list_empty(): 0.00 : ffff8000104680bc: cmp x19, x2 : blk_mq_try_issue_list_directly(): 0.00 : ffff8000104680c0: cset w2, eq // eq = none 0.00 : ffff8000104680c4: bl ffff800010467460 : break; 0.00 : ffff8000104680c8: b ffff800010468060 Percent | Source code & Disassembly of vmlinux for cycles (15 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011adc8 : : calc_load_nohz_start(): : atomic_long_add(delta, &calc_load_nohz[idx]); : } : } : : void calc_load_nohz_start(void) : { 0.00 : ffff80001011adc8: stp x29, x30, [sp, #-16]! : /* : * We're going into NO_HZ mode, if there's any pending delta, fold it : * into the pending NO_HZ delta. : */ : calc_load_nohz_fold(this_rq()); 0.00 : ffff80001011adcc: adrp x0, ffff800011528000 0.00 : ffff80001011add0: add x0, x0, #0xe80 : { 0.00 : ffff80001011add4: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 13.76 : ffff80001011add8: mrs x1, tpidr_el1 : calc_load_nohz_start(): : calc_load_nohz_fold(this_rq()); 0.00 : ffff80001011addc: add x0, x0, x1 0.00 : ffff80001011ade0: bl ffff80001011ac68 : } 86.24 : ffff80001011ade4: ldp x29, x30, [sp], #16 0.00 : ffff80001011ade8: ret Percent | Source code & Disassembly of vmlinux for cycles (33 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010467f70 : : blk_mq_request_issue_directly(): : : hctx_unlock(hctx, srcu_idx); : } : : blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last) : { 51.71 : ffff800010467f70: stp x29, x30, [sp, #-64]! 0.00 : ffff800010467f74: mov x29, sp 3.05 : ffff800010467f78: stp x21, x22, [sp, #32] 0.00 : ffff800010467f7c: mov x21, x0 3.07 : ffff800010467f80: stp x19, x20, [sp, #16] 0.00 : ffff800010467f84: adrp x19, ffff800011909000 0.00 : ffff800010467f88: add x19, x19, #0x908 0.00 : ffff800010467f8c: and w22, w1, #0xff : blk_status_t ret; : int srcu_idx; : blk_qc_t unused_cookie; : struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 3.08 : ffff800010467f90: ldr x20, [x21, #16] : { 3.06 : ffff800010467f94: ldr x0, [x19] 0.00 : ffff800010467f98: str x0, [x29, #56] 0.00 : ffff800010467f9c: mov x0, #0x0 // #0 : : hctx_lock(hctx, &srcu_idx); 0.00 : ffff800010467fa0: add x1, x29, #0x30 0.00 : ffff800010467fa4: mov x0, x20 17.93 : ffff800010467fa8: bl ffff8000104645b8 : ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true, last); 0.00 : ffff800010467fac: add x2, x29, #0x34 0.00 : ffff800010467fb0: mov x1, x21 0.00 : ffff800010467fb4: mov w4, w22 0.00 : ffff800010467fb8: mov w3, #0x1 // #1 5.92 : ffff800010467fbc: mov x0, x20 0.00 : ffff800010467fc0: bl ffff8000104674e0 <__blk_mq_try_issue_directly> : hctx_unlock(hctx, srcu_idx); 0.00 : ffff800010467fc4: ldr w1, [x29, #48] : ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true, last); 0.00 : ffff800010467fc8: mov w21, w0 : hctx_unlock(hctx, srcu_idx); 0.00 : ffff800010467fcc: mov x0, x20 0.00 : ffff800010467fd0: bl ffff800010464600 : : return ret; : } 0.00 : ffff800010467fd4: ldr x2, [x29, #56] 0.00 : ffff800010467fd8: ldr x1, [x19] 0.00 : ffff800010467fdc: eor x1, x2, x1 0.00 : ffff800010467fe0: cbnz x1, ffff800010467ff8 9.17 : ffff800010467fe4: mov w0, w21 3.01 : ffff800010467fe8: ldp x19, x20, [sp, #16] 0.00 : ffff800010467fec: ldp x21, x22, [sp, #32] 0.00 : ffff800010467ff0: ldp x29, x30, [sp], #64 0.00 : ffff800010467ff4: ret 0.00 : ffff800010467ff8: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (24 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101d7640 : : mempool_alloc(): : * Note: using __GFP_ZERO is not supported. : * : * Return: pointer to the allocated element or %NULL on error. : */ : void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) : { 3.87 : ffff8000101d7640: stp x29, x30, [sp, #-144]! : spin_unlock_irqrestore(&pool->lock, flags); : return NULL; : } : : /* Let's wait for someone else to return an element to @pool */ : init_wait(&wait); 0.00 : ffff8000101d7644: adrp x2, ffff800010130000 : { 0.00 : ffff8000101d7648: mov x29, sp 8.00 : ffff8000101d764c: stp x19, x20, [sp, #16] 0.00 : ffff8000101d7650: adrp x20, ffff800011909000 0.00 : ffff8000101d7654: stp x21, x22, [sp, #32] : gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO); 0.00 : ffff8000101d7658: mov w19, #0xfffffbbf // #-1089 : { 0.00 : ffff8000101d765c: stp x23, x24, [sp, #48] 0.00 : ffff8000101d7660: mov x21, x0 0.00 : ffff8000101d7664: stp x25, x26, [sp, #64] 0.00 : ffff8000101d7668: add x3, x20, #0x908 0.00 : ffff8000101d766c: str x27, [sp, #80] : gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO); 0.00 : ffff8000101d7670: and w19, w1, w19 : gfp_mask |= __GFP_NOWARN; /* failures are OK */ 0.00 : ffff8000101d7674: mov w0, #0x2000 // #8192 : might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 0.00 : ffff8000101d7678: and w24, w1, #0x400 : gfp_mask |= __GFP_NOWARN; /* failures are OK */ 8.36 : ffff8000101d767c: movk w0, #0x9, lsl #16 : init_wait(&wait); 0.00 : ffff8000101d7680: add x25, x2, #0x8 : gfp_mask |= __GFP_NOWARN; /* failures are OK */ 0.00 : ffff8000101d7684: orr w22, w1, w0 : gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO); 0.00 : ffff8000101d7688: orr w19, w19, w0 : prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 0.00 : ffff8000101d768c: add x26, x21, #0x30 : { 0.00 : ffff8000101d7690: ldr x4, [x3] 67.24 : ffff8000101d7694: str x4, [x29, #136] 0.00 : ffff8000101d7698: mov x4, #0x0 // #0 : INIT_LIST_HEAD(): : * Initializes the list_head to point to itself. If it is a list header, : * the result is an empty list. : */ : static inline void INIT_LIST_HEAD(struct list_head *list) : { : WRITE_ONCE(list->next, list); 0.00 : ffff8000101d769c: add x27, x29, #0x78 : mempool_alloc(): : element = pool->alloc(gfp_temp, pool->pool_data); 0.00 : ffff8000101d76a0: ldp x1, x2, [x21, #24] 0.00 : ffff8000101d76a4: mov w0, w19 0.00 : ffff8000101d76a8: blr x2 0.00 : ffff8000101d76ac: mov x23, x0 : if (likely(element != NULL)) 0.00 : ffff8000101d76b0: cbz x0, ffff8000101d76e8 : */ : io_schedule_timeout(5*HZ); : : finish_wait(&pool->wait, &wait); : goto repeat_alloc; : } 0.00 : ffff8000101d76b4: add x20, x20, #0x908 0.00 : ffff8000101d76b8: mov x0, x23 0.00 : ffff8000101d76bc: ldr x2, [x29, #136] 8.34 : ffff8000101d76c0: ldr x1, [x20] 0.00 : ffff8000101d76c4: eor x1, x2, x1 0.00 : ffff8000101d76c8: cbnz x1, ffff8000101d779c 0.00 : ffff8000101d76cc: ldp x19, x20, [sp, #16] 0.00 : ffff8000101d76d0: ldp x21, x22, [sp, #32] 0.00 : ffff8000101d76d4: ldp x23, x24, [sp, #48] 4.20 : ffff8000101d76d8: ldp x25, x26, [sp, #64] 0.00 : ffff8000101d76dc: ldr x27, [sp, #80] 0.00 : ffff8000101d76e0: ldp x29, x30, [sp], #144 0.00 : ffff8000101d76e4: ret 0.00 : ffff8000101d76e8: str x28, [x29, #88] : spin_lock_irqsave(&pool->lock, flags); 0.00 : ffff8000101d76ec: mov x0, x21 0.00 : ffff8000101d76f0: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff8000101d76f4: mov x28, x0 : if (likely(pool->curr_nr)) { 0.00 : ffff8000101d76f8: ldr w0, [x21, #8] 0.00 : ffff8000101d76fc: cbz w0, ffff8000101d772c : remove_element(): : void *element = pool->elements[--pool->curr_nr]; 0.00 : ffff8000101d7700: sub w0, w0, #0x1 : mempool_alloc(): : element = remove_element(pool); 0.00 : ffff8000101d7704: ldr x1, [x21, #16] : remove_element(): : void *element = pool->elements[--pool->curr_nr]; 0.00 : ffff8000101d7708: str w0, [x21, #8] 0.00 : ffff8000101d770c: ldr x23, [x1, w0, sxtw #3] : BUG_ON(pool->curr_nr < 0); 0.00 : ffff8000101d7710: tbnz w0, #31, ffff8000101d774c : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000101d7714: mov x1, x28 0.00 : ffff8000101d7718: mov x0, x21 0.00 : ffff8000101d771c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : mempool_alloc(): : smp_wmb(); 0.00 : ffff8000101d7720: dmb ishst : return element; 0.00 : ffff8000101d7724: ldr x28, [x29, #88] 0.00 : ffff8000101d7728: b ffff8000101d76b4 : if (gfp_temp != gfp_mask) { 0.00 : ffff8000101d772c: cmp w19, w22 0.00 : ffff8000101d7730: b.eq ffff8000101d7750 // b.none : spin_unlock_irqrestore(): 0.00 : ffff8000101d7734: mov x1, x28 0.00 : ffff8000101d7738: mov x0, x21 0.00 : ffff8000101d773c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : mempool_alloc(): : gfp_mask |= __GFP_NOWARN; /* failures are OK */ 0.00 : ffff8000101d7740: mov w19, w22 0.00 : ffff8000101d7744: ldr x28, [x29, #88] 0.00 : ffff8000101d7748: b ffff8000101d76a0 : remove_element(): 0.00 : ffff8000101d774c: bl ffff8000101d7638 : mempool_alloc(): : if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { 0.00 : ffff8000101d7750: cbz w24, ffff8000101d77a4 : prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 0.00 : ffff8000101d7754: mov w2, #0x2 // #2 0.00 : ffff8000101d7758: add x1, x29, #0x60 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000101d775c: mrs x0, sp_el0 : mempool_alloc(): : init_wait(&wait); 0.00 : ffff8000101d7760: str wzr, [x29, #96] 0.00 : ffff8000101d7764: stp x0, x25, [x29, #104] : prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 0.00 : ffff8000101d7768: mov x0, x26 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101d776c: str x27, [x29, #120] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff8000101d7770: str x27, [x29, #128] : mempool_alloc(): 0.00 : ffff8000101d7774: bl ffff80001012fa78 : spin_unlock_irqrestore(): 0.00 : ffff8000101d7778: mov x1, x28 0.00 : ffff8000101d777c: mov x0, x21 0.00 : ffff8000101d7780: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : mempool_alloc(): : io_schedule_timeout(5*HZ); 0.00 : ffff8000101d7784: mov x0, #0x4e2 // #1250 0.00 : ffff8000101d7788: bl ffff800010cd2fc8 : finish_wait(&pool->wait, &wait); 0.00 : ffff8000101d778c: add x1, x29, #0x60 0.00 : ffff8000101d7790: mov x0, x26 0.00 : ffff8000101d7794: bl ffff80001012fc00 : goto repeat_alloc; 0.00 : ffff8000101d7798: b ffff8000101d7740 0.00 : ffff8000101d779c: str x28, [x29, #88] : } 0.00 : ffff8000101d77a0: bl ffff8000100e64d0 <__stack_chk_fail> : spin_unlock_irqrestore(): 0.00 : ffff8000101d77a4: mov x1, x28 0.00 : ffff8000101d77a8: mov x0, x21 0.00 : ffff8000101d77ac: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : mempool_alloc(): : return NULL; 0.00 : ffff8000101d77b0: ldr x28, [x29, #88] 0.00 : ffff8000101d77b4: b ffff8000101d76b4 Percent | Source code & Disassembly of vmlinux for cycles (29 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010169590 : : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 7.00 : ffff800010169590: mrs x2, tpidr_el1 : round_jiffies_up(): : * of firing does not matter too much, as long as they don't fire too : * early. : */ : unsigned long round_jiffies_up(unsigned long j) : { : return round_jiffies_common(j, raw_smp_processor_id(), true); 6.76 : ffff800010169594: adrp x1, ffff80001151d000 0.00 : ffff800010169598: add x1, x1, #0x18 : round_jiffies_common(): : j += cpu * 3; 0.00 : ffff80001016959c: ldr w2, [x1, x2] : rem = j % HZ; 13.88 : ffff8000101695a0: mov x5, #0xf7cf // #63439 : return time_is_after_jiffies(j) ? j : original; 6.71 : ffff8000101695a4: adrp x1, ffff800011907000 : rem = j % HZ; 0.00 : ffff8000101695a8: movk x5, #0xe353, lsl #16 55.24 : ffff8000101695ac: movk x5, #0x9ba5, lsl #32 : j -= cpu * 3; 0.00 : ffff8000101695b0: add x3, x0, #0xfa : j += cpu * 3; 0.00 : ffff8000101695b4: add w2, w2, w2, lsl #1 : return time_is_after_jiffies(j) ? j : original; 3.50 : ffff8000101695b8: ldr x4, [x1, #2432] : rem = j % HZ; 0.00 : ffff8000101695bc: movk x5, #0x20c4, lsl #48 : j += cpu * 3; 0.00 : ffff8000101695c0: add x2, x0, w2, sxtw : rem = j % HZ; 0.00 : ffff8000101695c4: lsr x1, x2, #1 0.00 : ffff8000101695c8: umulh x1, x1, x5 0.00 : ffff8000101695cc: lsr x1, x1, #4 0.00 : ffff8000101695d0: lsl x5, x1, #5 0.00 : ffff8000101695d4: sub x5, x5, x1 0.00 : ffff8000101695d8: add x1, x1, x5, lsl #2 0.00 : ffff8000101695dc: sub x1, x2, x1, lsl #1 : j -= cpu * 3; 0.00 : ffff8000101695e0: sub x1, x3, x1 : return time_is_after_jiffies(j) ? j : original; 0.00 : ffff8000101695e4: sub x2, x4, x1 6.91 : ffff8000101695e8: cmp x2, #0x0 : round_jiffies_up(): : } 0.00 : ffff8000101695ec: csel x0, x1, x0, lt // lt = tstop 0.00 : ffff8000101695f0: ret Percent | Source code & Disassembly of vmlinux for cycles (36 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010466d00 : : blk_mq_get_driver_tag(): : : return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); : } : : bool blk_mq_get_driver_tag(struct request *rq) : { 2.87 : ffff800010466d00: stp x29, x30, [sp, #-96]! 0.00 : ffff800010466d04: mov x29, sp : struct blk_mq_alloc_data data = { 0.00 : ffff800010466d08: add x1, x29, #0x3c : { 0.00 : ffff800010466d0c: stp x19, x20, [sp, #16] 0.00 : ffff800010466d10: mov x19, x0 0.00 : ffff800010466d14: adrp x20, ffff800011909000 0.00 : ffff800010466d18: add x0, x20, #0x908 : struct blk_mq_alloc_data data = { 0.00 : ffff800010466d1c: stp xzr, xzr, [x1] : { 2.82 : ffff800010466d20: ldr x1, [x0] 2.80 : ffff800010466d24: str x1, [x29, #88] 0.00 : ffff800010466d28: mov x1, #0x0 // #0 : struct blk_mq_alloc_data data = { 2.80 : ffff800010466d2c: ldr w3, [x19, #24] 0.00 : ffff800010466d30: mov w0, #0x1 // #1 0.00 : ffff800010466d34: ldr x4, [x19] : .q = rq->q, : .hctx = rq->mq_hctx, 5.55 : ffff800010466d38: ldr x1, [x19, #16] : .flags = BLK_MQ_REQ_NOWAIT, : .cmd_flags = rq->cmd_flags, : }; : bool shared; : : if (rq->tag != -1) 5.43 : ffff800010466d3c: ldr w2, [x19, #32] : struct blk_mq_alloc_data data = { 5.65 : ffff800010466d40: str x4, [x29, #48] 25.11 : ffff800010466d44: str w0, [x29, #56] : if (rq->tag != -1) 0.00 : ffff800010466d48: cmn w2, #0x1 : struct blk_mq_alloc_data data = { 0.00 : ffff800010466d4c: str w3, [x29, #64] 5.66 : ffff800010466d50: str wzr, [x29, #76] 0.00 : ffff800010466d54: str x1, [x29, #80] : if (rq->tag != -1) 0.00 : ffff800010466d58: b.eq ffff800010466d7c // b.none : } : data.hctx->tags->rqs[rq->tag] = rq; : } : : return rq->tag != -1; : } 5.33 : ffff800010466d5c: add x20, x20, #0x908 0.00 : ffff800010466d60: ldr x2, [x29, #88] 0.00 : ffff800010466d64: ldr x1, [x20] 0.00 : ffff800010466d68: eor x1, x2, x1 0.00 : ffff800010466d6c: cbnz x1, ffff800010466e54 2.78 : ffff800010466d70: ldp x19, x20, [sp, #16] 33.20 : ffff800010466d74: ldp x29, x30, [sp], #96 0.00 : ffff800010466d78: ret : if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag)) 0.00 : ffff800010466d7c: ldr x0, [x1, #344] 0.00 : ffff800010466d80: ldr w2, [x19, #36] 0.00 : ffff800010466d84: ldr w0, [x0, #4] 0.00 : ffff800010466d88: cmp w2, w0 0.00 : ffff800010466d8c: b.cc ffff800010466dc8 // b.lo, b.ul, b.last : blk_mq_tag_busy(): : extern bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *); : extern void __blk_mq_tag_idle(struct blk_mq_hw_ctx *); : : static inline bool blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) : { : if (!(hctx->flags & BLK_MQ_F_TAG_SHARED)) 0.00 : ffff800010466d90: ldr x0, [x1, #192] 0.00 : ffff800010466d94: tbnz w0, #1, ffff800010466dd8 : blk_mq_get_driver_tag(): : rq->tag = blk_mq_get_tag(&data); 0.00 : ffff800010466d98: add x0, x29, #0x30 0.00 : ffff800010466d9c: bl ffff80001046a5d0 0.00 : ffff800010466da0: str w0, [x19, #32] : if (rq->tag >= 0) { 0.00 : ffff800010466da4: tbnz w0, #31, ffff800010466dbc 0.00 : ffff800010466da8: ldr x1, [x29, #80] : data.hctx->tags->rqs[rq->tag] = rq; 0.00 : ffff800010466dac: ldr x1, [x1, #336] 0.00 : ffff800010466db0: ldr x1, [x1, #144] 0.00 : ffff800010466db4: str x19, [x1, w0, sxtw #3] 0.00 : ffff800010466db8: ldr w0, [x19, #32] : return rq->tag != -1; 0.00 : ffff800010466dbc: cmn w0, #0x1 0.00 : ffff800010466dc0: cset w0, ne // ne = any 0.00 : ffff800010466dc4: b ffff800010466d5c : data.flags |= BLK_MQ_REQ_RESERVED; 0.00 : ffff800010466dc8: mov w0, #0x3 // #3 0.00 : ffff800010466dcc: str w0, [x29, #56] : blk_mq_tag_busy(): 0.00 : ffff800010466dd0: ldr x0, [x1, #192] 0.00 : ffff800010466dd4: tbz w0, #1, ffff800010466d98 : return false; : : return __blk_mq_tag_busy(hctx); 0.00 : ffff800010466dd8: mov x0, x1 0.00 : ffff800010466ddc: str x21, [x29, #32] 0.00 : ffff800010466de0: bl ffff80001046a440 <__blk_mq_tag_busy> 0.00 : ffff800010466de4: and w21, w0, #0xff : blk_mq_get_driver_tag(): : rq->tag = blk_mq_get_tag(&data); 0.00 : ffff800010466de8: add x0, x29, #0x30 0.00 : ffff800010466dec: bl ffff80001046a5d0 0.00 : ffff800010466df0: str w0, [x19, #32] : if (rq->tag >= 0) { 0.00 : ffff800010466df4: tbnz w0, #31, ffff800010466e44 0.00 : ffff800010466df8: ldr x1, [x29, #80] : if (shared) { 0.00 : ffff800010466dfc: cbz w21, ffff800010466e4c : rq->rq_flags |= RQF_MQ_INFLIGHT; 0.00 : ffff800010466e00: ldr w0, [x19, #28] : atomic_inc(&data.hctx->nr_active); 0.00 : ffff800010466e04: add x1, x1, #0x1b0 : rq->rq_flags |= RQF_MQ_INFLIGHT; 0.00 : ffff800010466e08: orr w0, w0, #0x40 0.00 : ffff800010466e0c: str w0, [x19, #28] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010466e10: b ffff800010466e30 0.00 : ffff800010466e14: b ffff800010466e30 : __lse_atomic_add(): : } : : ATOMIC_OP(andnot, stclr) : ATOMIC_OP(or, stset) : ATOMIC_OP(xor, steor) : ATOMIC_OP(add, stadd) 0.00 : ffff800010466e18: mov w0, #0x1 // #1 0.00 : ffff800010466e1c: stadd w0, [x1] 0.00 : ffff800010466e20: ldr w0, [x19, #32] 0.00 : ffff800010466e24: ldr x21, [x29, #32] 0.00 : ffff800010466e28: ldr x1, [x29, #80] 0.00 : ffff800010466e2c: b ffff800010466dac : __ll_sc_atomic_add(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff800010466e30: b ffff800010469f34 0.00 : ffff800010466e34: ldr w0, [x19, #32] 0.00 : ffff800010466e38: ldr x21, [x29, #32] 0.00 : ffff800010466e3c: ldr x1, [x29, #80] 0.00 : ffff800010466e40: b ffff800010466dac 0.00 : ffff800010466e44: ldr x21, [x29, #32] 0.00 : ffff800010466e48: b ffff800010466dbc 0.00 : ffff800010466e4c: ldr x21, [x29, #32] 0.00 : ffff800010466e50: b ffff800010466dac 0.00 : ffff800010466e54: str x21, [x29, #32] : blk_mq_get_driver_tag(): : } 0.00 : ffff800010466e58: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (13 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001026d8d8 : : memcg_kmem_put_cache(): : * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache : * @cachep: the cache returned by memcg_kmem_get_cache : */ : void memcg_kmem_put_cache(struct kmem_cache *cachep) : { : if (!is_root_cache(cachep)) 0.00 : ffff80001026d8d8: ldr x1, [x0, #208] 61.91 : ffff80001026d8dc: cbz x1, ffff80001026d8fc : { 0.00 : ffff80001026d8e0: stp x29, x30, [sp, #-16]! : percpu_ref_put(): : * : * This function is safe to call as long as @ref is between init and exit. : */ : static inline void percpu_ref_put(struct percpu_ref *ref) : { : percpu_ref_put_many(ref, 1); 0.00 : ffff80001026d8e4: mov x1, #0x1 // #1 0.00 : ffff80001026d8e8: add x0, x0, #0x100 : memcg_kmem_put_cache(): 0.00 : ffff80001026d8ec: mov x29, sp : percpu_ref_put(): 0.00 : ffff80001026d8f0: bl ffff800010269108 : memcg_kmem_put_cache(): : percpu_ref_put(&cachep->memcg_params.refcnt); : } 0.00 : ffff80001026d8f4: ldp x29, x30, [sp], #16 0.00 : ffff80001026d8f8: ret 38.09 : ffff80001026d8fc: ret Percent | Source code & Disassembly of vmlinux for cycles (23 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104802c0 : : bio_integrity_prep(): : * to calling. In the WRITE case, integrity metadata will be generated using : * the block device's integrity function. In the READ case, the buffer : * will be prepared for DMA and a suitable end_io handler set up. : */ : bool bio_integrity_prep(struct bio *bio) : { 3.98 : ffff8000104802c0: stp x29, x30, [sp, #-96]! 0.00 : ffff8000104802c4: mov x29, sp 8.80 : ffff8000104802c8: str x26, [sp, #72] : struct bio_integrity_payload *bip; : struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); 0.00 : ffff8000104802cc: ldr x1, [x0, #8] 4.48 : ffff8000104802d0: ldr x26, [x1, #1040] : blk_get_integrity(): : : static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk) : { : struct blk_integrity *bi = &disk->queue->integrity; : : if (!bi->profile) 0.00 : ffff8000104802d4: mov x1, x26 43.61 : ffff8000104802d8: ldr x2, [x1, #200]! : bio_integrity_prep(): : unsigned int len, nr_pages; : unsigned int bytes, offset, i; : unsigned int intervals; : blk_status_t status; : : if (!bi) 0.00 : ffff8000104802dc: cmp x1, #0x0 : return true; 12.67 : ffff8000104802e0: mov w1, #0x1 // #1 : if (!bi) 0.00 : ffff8000104802e4: ccmp x2, #0x0, #0x4, ne // ne = any 22.05 : ffff8000104802e8: b.eq ffff800010480300 // b.none : : if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE) 0.00 : ffff8000104802ec: ldr w3, [x0, #16] : return true; 0.00 : ffff8000104802f0: mov w1, #0x1 // #1 : if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE) 0.00 : ffff8000104802f4: and w4, w3, #0xff 0.00 : ffff8000104802f8: cmp w4, w1 0.00 : ffff8000104802fc: b.ls ffff800010480310 // b.plast : err_end_io: : bio->bi_status = status; : bio_endio(bio); : return false; : : } 0.00 : ffff800010480300: mov w0, w1 4.40 : ffff800010480304: ldr x26, [sp, #72] 0.00 : ffff800010480308: ldp x29, x30, [sp], #96 0.00 : ffff80001048030c: ret : if (!bio_sectors(bio)) 0.00 : ffff800010480310: ldr w4, [x0, #40] 0.00 : ffff800010480314: lsr w4, w4, #9 0.00 : ffff800010480318: cbz w4, ffff800010480300 : bio_integrity(): : : #if defined(CONFIG_BLK_DEV_INTEGRITY) : : static inline struct bio_integrity_payload *bio_integrity(struct bio *bio) : { : if (bio->bi_opf & REQ_INTEGRITY) 0.00 : ffff80001048031c: tbz w3, #16, ffff800010480328 : bio_integrity_prep(): : if (bio_integrity(bio)) 0.00 : ffff800010480320: ldr x5, [x0, #88] 0.00 : ffff800010480324: cbnz x5, ffff800010480300 : if (bio_data_dir(bio) == READ) { 0.00 : ffff800010480328: tbnz w3, #0, ffff800010480344 : if (!bi->profile->verify_fn || 0.00 : ffff80001048032c: ldr x2, [x2, #8] : return true; 0.00 : ffff800010480330: mov w1, #0x1 // #1 : if (!bi->profile->verify_fn || 0.00 : ffff800010480334: cbz x2, ffff800010480300 0.00 : ffff800010480338: ldrb w2, [x26, #208] 0.00 : ffff80001048033c: tbnz w2, #0, ffff800010480358 0.00 : ffff800010480340: b ffff800010480300 : if (!bi->profile->generate_fn || 0.00 : ffff800010480344: ldr x2, [x2] : return true; 0.00 : ffff800010480348: mov w1, #0x1 // #1 : if (!bi->profile->generate_fn || 0.00 : ffff80001048034c: cbz x2, ffff800010480300 0.00 : ffff800010480350: ldrb w2, [x26, #208] 0.00 : ffff800010480354: tbz w2, #1, ffff800010480300 0.00 : ffff800010480358: stp x19, x20, [x29, #16] 0.00 : ffff80001048035c: mov x19, x0 0.00 : ffff800010480360: str x21, [x29, #32] 0.00 : ffff800010480364: str x27, [x29, #80] : bio_integrity_intervals(): : * to the appropriate number of integrity intervals. : */ : static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, : unsigned int sectors) : { : return sectors >> (bi->interval_exp - 9); 0.00 : ffff800010480368: ldrb w20, [x26, #210] : bio_integrity_prep(): : len = intervals * bi->tuple_size; 0.00 : ffff80001048036c: ldrb w0, [x26, #209] : bio_integrity_intervals(): 0.00 : ffff800010480370: sub w20, w20, #0x9 : bio_integrity_prep(): : buf = kmalloc(len, GFP_NOIO | q->bounce_gfp); 0.00 : ffff800010480374: ldr w1, [x26, #120] : bio_integrity_intervals(): 0.00 : ffff800010480378: lsr w4, w4, w20 : kmalloc(): : return kmem_cache_alloc_trace( : kmalloc_caches[kmalloc_type(flags)][index], : flags, size); : #endif : } : return __kmalloc(size, flags); 0.00 : ffff80001048037c: orr w1, w1, #0xc00 : bio_integrity_prep(): : len = intervals * bi->tuple_size; 0.00 : ffff800010480380: mul w21, w4, w0 : kmalloc(): 0.00 : ffff800010480384: mov x0, x21 : bio_integrity_prep(): 0.00 : ffff800010480388: mov x20, x21 : kmalloc(): 0.00 : ffff80001048038c: bl ffff800010252fd8 <__kmalloc> 0.00 : ffff800010480390: mov x27, x0 : bio_integrity_prep(): : if (unlikely(buf == NULL)) { 0.00 : ffff800010480394: cbz x0, ffff8000104804d8 : end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 0.00 : ffff800010480398: add x2, x21, #0xfff 0.00 : ffff80001048039c: stp x22, x23, [x29, #40] 0.00 : ffff8000104803a0: add x2, x2, x0 : start = ((unsigned long) buf) >> PAGE_SHIFT; 0.00 : ffff8000104803a4: lsr x22, x0, #12 : bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages); 0.00 : ffff8000104803a8: mov w1, #0xc00 // #3072 0.00 : ffff8000104803ac: mov x0, x19 : end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 0.00 : ffff8000104803b0: lsr x2, x2, #12 : nr_pages = end - start; 0.00 : ffff8000104803b4: sub w22, w2, w22 : bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages); 0.00 : ffff8000104803b8: mov w2, w22 0.00 : ffff8000104803bc: bl ffff800010480148 0.00 : ffff8000104803c0: mov x23, x0 : if (IS_ERR(bip)) { 0.00 : ffff8000104803c4: cmn x0, #0x1, lsl #12 0.00 : ffff8000104803c8: b.hi ffff800010480508 // b.pmore : bip->bip_flags |= BIP_BLOCK_INTEGRITY; 0.00 : ffff8000104803cc: ldrh w0, [x0, #38] : bip->bip_iter.bi_size = len; 0.00 : ffff8000104803d0: str w21, [x23, #16] : bip->bip_flags |= BIP_BLOCK_INTEGRITY; 0.00 : ffff8000104803d4: orr w1, w0, #0x1 0.00 : ffff8000104803d8: strh w1, [x23, #38] : bip_set_seed(bip, bio->bi_iter.bi_sector); 0.00 : ffff8000104803dc: ldr x1, [x19, #32] : bip_set_seed(): : } : : static inline void bip_set_seed(struct bio_integrity_payload *bip, : sector_t seed) : { : bip->bip_iter.bi_sector = seed; 0.00 : ffff8000104803e0: str x1, [x23, #8] : bio_integrity_prep(): : if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM) 0.00 : ffff8000104803e4: ldrb w1, [x26, #208] 0.00 : ffff8000104803e8: tbz w1, #3, ffff8000104803f8 : bip->bip_flags |= BIP_IP_CHECKSUM; 0.00 : ffff8000104803ec: mov w1, #0x11 // #17 0.00 : ffff8000104803f0: orr w0, w0, w1 0.00 : ffff8000104803f4: strh w0, [x23, #38] : offset = offset_in_page(buf); 0.00 : ffff8000104803f8: and w3, w27, #0xfff : for (i = 0 ; i < nr_pages ; i++) { 0.00 : ffff8000104803fc: cbz w22, ffff80001048047c : bytes = PAGE_SIZE - offset; 0.00 : ffff800010480400: mov w21, #0x1000 // #4096 0.00 : ffff800010480404: sub w21, w21, w3 : if (len <= 0) 0.00 : ffff800010480408: cbz w20, ffff80001048047c 0.00 : ffff80001048040c: stp x24, x25, [x29, #56] : ret = bio_integrity_add_page(bio, virt_to_page(buf), 0.00 : ffff800010480410: mov x24, #0xffffffffffe00000 // #-2097152 0.00 : ffff800010480414: str x28, [x29, #88] 0.00 : ffff800010480418: mov x25, #0x1000000000000 // #281474976710656 0.00 : ffff80001048041c: movk x24, #0xfdff, lsl #32 : for (i = 0 ; i < nr_pages ; i++) { 0.00 : ffff800010480420: mov w28, #0x0 // #0 0.00 : ffff800010480424: b ffff800010480444 : len -= bytes; 0.00 : ffff800010480428: sub w20, w20, w21 : buf += bytes; 0.00 : ffff80001048042c: add x27, x27, w21, uxtw : for (i = 0 ; i < nr_pages ; i++) { 0.00 : ffff800010480430: cmp w22, w28 0.00 : ffff800010480434: b.eq ffff800010480474 // b.none : offset = 0; 0.00 : ffff800010480438: mov w3, #0x0 // #0 : bytes = PAGE_SIZE - offset; 0.00 : ffff80001048043c: mov w21, #0x1000 // #4096 : if (len <= 0) 0.00 : ffff800010480440: cbz w20, ffff800010480474 : ret = bio_integrity_add_page(bio, virt_to_page(buf), 0.00 : ffff800010480444: add x1, x27, x25 0.00 : ffff800010480448: cmp w20, w21 0.00 : ffff80001048044c: csel w21, w20, w21, ls // ls = plast : for (i = 0 ; i < nr_pages ; i++) { 0.00 : ffff800010480450: add w28, w28, #0x1 : ret = bio_integrity_add_page(bio, virt_to_page(buf), 0.00 : ffff800010480454: lsr x1, x1, #12 0.00 : ffff800010480458: mov w2, w21 0.00 : ffff80001048045c: mov x0, x19 0.00 : ffff800010480460: add x1, x24, x1, lsl #6 0.00 : ffff800010480464: bl ffff80001047fd70 : if (ret == 0) { 0.00 : ffff800010480468: cbz w0, ffff800010480524 : if (ret < bytes) 0.00 : ffff80001048046c: cmp w0, w21 0.00 : ffff800010480470: b.cs ffff800010480428 // b.hs, b.nlast 0.00 : ffff800010480474: ldp x24, x25, [x29, #56] 0.00 : ffff800010480478: ldr x28, [x29, #88] : op_is_write(): : bio->bi_opf = op | op_flags; : } : : static inline bool op_is_write(unsigned int op) : { : return (op & 1); 0.00 : ffff80001048047c: ldr w0, [x19, #16] : bio_integrity_prep(): : if (bio_data_dir(bio) == WRITE) { 0.00 : ffff800010480480: tbnz w0, #0, ffff8000104804ac : bip->bio_iter = bio->bi_iter; 0.00 : ffff800010480484: ldp x2, x3, [x19, #32] 0.00 : ffff800010480488: stp x2, x3, [x23, #40] : return true; 0.00 : ffff80001048048c: mov w1, #0x1 // #1 : bip->bio_iter = bio->bi_iter; 0.00 : ffff800010480490: ldr x0, [x19, #48] 0.00 : ffff800010480494: str x0, [x23, #56] 0.00 : ffff800010480498: ldp x19, x20, [x29, #16] 0.00 : ffff80001048049c: ldp x21, x22, [x29, #32] 0.00 : ffff8000104804a0: ldr x23, [x29, #48] 0.00 : ffff8000104804a4: ldr x27, [x29, #80] 0.00 : ffff8000104804a8: b ffff800010480300 : bio_integrity_process(bio, &bio->bi_iter, 0.00 : ffff8000104804ac: ldr x2, [x26, #200] 0.00 : ffff8000104804b0: add x1, x19, #0x20 0.00 : ffff8000104804b4: mov x0, x19 0.00 : ffff8000104804b8: ldr x2, [x2] 0.00 : ffff8000104804bc: bl ffff80001047feb0 0.00 : ffff8000104804c0: ldr x23, [x29, #48] : return true; 0.00 : ffff8000104804c4: mov w1, #0x1 // #1 0.00 : ffff8000104804c8: ldr x27, [x29, #80] 0.00 : ffff8000104804cc: ldp x19, x20, [x29, #16] 0.00 : ffff8000104804d0: ldp x21, x22, [x29, #32] 0.00 : ffff8000104804d4: b ffff800010480300 : printk(KERN_ERR "could not allocate integrity buffer\n"); 0.00 : ffff8000104804d8: adrp x0, ffff8000111eb000 0.00 : ffff8000104804dc: add x0, x0, #0x238 0.00 : ffff8000104804e0: bl ffff80001014a85c : bio->bi_status = status; 0.00 : ffff8000104804e4: mov w0, #0x9 // #9 0.00 : ffff8000104804e8: strb w0, [x19, #26] : bio_endio(bio); 0.00 : ffff8000104804ec: mov x0, x19 0.00 : ffff8000104804f0: bl ffff800010456b18 : return false; 0.00 : ffff8000104804f4: ldr x21, [x29, #32] 0.00 : ffff8000104804f8: mov w1, #0x0 // #0 0.00 : ffff8000104804fc: ldr x27, [x29, #80] 0.00 : ffff800010480500: ldp x19, x20, [x29, #16] 0.00 : ffff800010480504: b ffff800010480300 : printk(KERN_ERR "could not allocate data integrity bioset\n"); 0.00 : ffff800010480508: adrp x0, ffff8000111eb000 0.00 : ffff80001048050c: add x0, x0, #0x260 0.00 : ffff800010480510: bl ffff80001014a85c : kfree(buf); 0.00 : ffff800010480514: mov x0, x27 0.00 : ffff800010480518: bl ffff800010253890 : goto err_end_io; 0.00 : ffff80001048051c: ldp x22, x23, [x29, #40] 0.00 : ffff800010480520: b ffff8000104804e4 : printk(KERN_ERR "could not attach integrity payload\n"); 0.00 : ffff800010480524: adrp x0, ffff8000111eb000 0.00 : ffff800010480528: add x0, x0, #0x290 0.00 : ffff80001048052c: bl ffff80001014a85c : kfree(buf); 0.00 : ffff800010480530: mov x0, x27 0.00 : ffff800010480534: bl ffff800010253890 : goto err_end_io; 0.00 : ffff800010480538: ldp x22, x23, [x29, #40] 0.00 : ffff80001048053c: ldp x24, x25, [x29, #56] 0.00 : ffff800010480540: ldr x28, [x29, #88] 0.00 : ffff800010480544: b ffff8000104804e4 Percent | Source code & Disassembly of vmlinux for cycles (41 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010439108 : : security_file_permission(): : { : return call_int_hook(kernfs_init_security, 0, kn_dir, kn); : } : : int security_file_permission(struct file *file, int mask) : { 0.00 : ffff800010439108: stp x29, x30, [sp, #-64]! : int ret; : : ret = call_int_hook(file_permission, 0, file, mask); 0.00 : ffff80001043910c: adrp x2, ffff8000112f3000 : { 0.00 : ffff800010439110: mov x29, sp 0.00 : ffff800010439114: stp x19, x20, [sp, #16] 0.00 : ffff800010439118: mov x20, x0 : ret = call_int_hook(file_permission, 0, file, mask); 0.00 : ffff80001043911c: ldr x19, [x2, #56] : { 2.50 : ffff800010439120: str x21, [sp, #32] 0.00 : ffff800010439124: mov w21, w1 : ret = call_int_hook(file_permission, 0, file, mask); 0.00 : ffff800010439128: cbz x19, ffff80001043914c 0.00 : ffff80001043912c: nop 0.00 : ffff800010439130: ldr x2, [x19, #24] 0.00 : ffff800010439134: mov w1, w21 0.00 : ffff800010439138: mov x0, x20 0.00 : ffff80001043913c: blr x2 0.00 : ffff800010439140: cbnz w0, ffff8000104391a8 0.00 : ffff800010439144: ldr x19, [x19] 0.00 : ffff800010439148: cbnz x19, ffff800010439130 : fsnotify_perm(): : int ret; : const struct path *path = &file->f_path; : struct inode *inode = file_inode(file); : __u32 fsnotify_mask = 0; : : if (file->f_mode & FMODE_NONOTIFY) 0.00 : ffff80001043914c: ldr w1, [x20, #68] : return 0; : if (!(mask & (MAY_READ | MAY_OPEN))) : return 0; 0.00 : ffff800010439150: mov w0, #0x0 // #0 : if (file->f_mode & FMODE_NONOTIFY) 0.00 : ffff800010439154: tbnz w1, #26, ffff8000104391a8 : if (!(mask & (MAY_READ | MAY_OPEN))) 21.71 : ffff800010439158: mov w0, #0x24 // #36 0.00 : ffff80001043915c: ands w0, w21, w0 0.00 : ffff800010439160: b.eq ffff8000104391a8 // b.none 2.48 : ffff800010439164: stp x22, x23, [x29, #40] : const struct path *path = &file->f_path; 0.00 : ffff800010439168: add x22, x20, #0x10 : struct inode *inode = file_inode(file); 0.00 : ffff80001043916c: ldp x1, x23, [x20, #24] : if (mask & MAY_OPEN) { 0.00 : ffff800010439170: tbz w21, #5, ffff8000104391b8 : fsnotify_mask = FS_OPEN_PERM; : : if (file->f_flags & __FMODE_EXEC) { 0.00 : ffff800010439174: ldr w0, [x20, #64] : fsnotify_mask = FS_OPEN_PERM; 0.00 : ffff800010439178: mov w19, #0x10000 // #65536 : if (file->f_flags & __FMODE_EXEC) { 0.00 : ffff80001043917c: tbnz w0, #5, ffff8000104391e8 : } : } else if (mask & MAY_READ) { : fsnotify_mask = FS_ACCESS_PERM; : } : : if (S_ISDIR(inode->i_mode)) 0.00 : ffff800010439180: ldrh w2, [x23] : fsnotify_mask |= FS_ISDIR; 0.00 : ffff800010439184: orr w3, w19, #0x40000000 : fsnotify_parent(): : return __fsnotify_parent(path, dentry, mask); 0.00 : ffff800010439188: mov x0, x22 : fsnotify_perm(): : if (S_ISDIR(inode->i_mode)) 0.00 : ffff80001043918c: and w2, w2, #0xf000 : fsnotify_mask |= FS_ISDIR; 41.67 : ffff800010439190: cmp w2, #0x4, lsl #12 0.00 : ffff800010439194: csel w19, w3, w19, eq // eq = none : fsnotify_parent(): : return __fsnotify_parent(path, dentry, mask); 0.00 : ffff800010439198: mov w2, w19 0.00 : ffff80001043919c: bl ffff8000102cd4c8 <__fsnotify_parent> : fsnotify_path(): : if (ret) 0.00 : ffff8000104391a0: cbz w0, ffff8000104391c4 0.00 : ffff8000104391a4: ldp x22, x23, [x29, #40] : security_file_permission(): : if (ret) : return ret; : : return fsnotify_perm(file, mask); : } 24.32 : ffff8000104391a8: ldp x19, x20, [sp, #16] 0.00 : ffff8000104391ac: ldr x21, [sp, #32] 0.00 : ffff8000104391b0: ldp x29, x30, [sp], #64 0.00 : ffff8000104391b4: ret : fsnotify_perm(): : __u32 fsnotify_mask = 0; 2.44 : ffff8000104391b8: ubfx x19, x21, #2, #1 0.00 : ffff8000104391bc: lsl w19, w19, #17 0.00 : ffff8000104391c0: b ffff800010439180 : fsnotify_path(): : return fsnotify(inode, mask, path, FSNOTIFY_EVENT_PATH, NULL, 0); 0.00 : ffff8000104391c4: mov x2, x22 0.00 : ffff8000104391c8: mov x0, x23 0.00 : ffff8000104391cc: mov w5, #0x0 // #0 0.00 : ffff8000104391d0: mov x4, #0x0 // #0 0.00 : ffff8000104391d4: mov w3, #0x1 // #1 0.00 : ffff8000104391d8: mov w1, w19 0.00 : ffff8000104391dc: bl ffff8000102cd188 4.88 : ffff8000104391e0: ldp x22, x23, [x29, #40] 0.00 : ffff8000104391e4: b ffff8000104391a8 : fsnotify_parent(): : return __fsnotify_parent(path, dentry, mask); 0.00 : ffff8000104391e8: mov w2, #0x40000 // #262144 0.00 : ffff8000104391ec: mov x0, x22 0.00 : ffff8000104391f0: bl ffff8000102cd4c8 <__fsnotify_parent> : fsnotify_path(): : if (ret) 0.00 : ffff8000104391f4: cbnz w0, ffff8000104391a4 : return fsnotify(inode, mask, path, FSNOTIFY_EVENT_PATH, NULL, 0); 0.00 : ffff8000104391f8: mov w5, #0x0 // #0 0.00 : ffff8000104391fc: mov x4, #0x0 // #0 0.00 : ffff800010439200: mov w3, #0x1 // #1 0.00 : ffff800010439204: mov x2, x22 0.00 : ffff800010439208: mov w1, #0x40000 // #262144 0.00 : ffff80001043920c: mov x0, x23 0.00 : ffff800010439210: bl ffff8000102cd188 : fsnotify_perm(): : if (ret) 0.00 : ffff800010439214: cbnz w0, ffff8000104391a4 0.00 : ffff800010439218: ldr x1, [x20, #24] 0.00 : ffff80001043921c: b ffff800010439180 Percent | Source code & Disassembly of vmlinux for cycles (26 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015df08 : : rcu_qs(): : * Callers to this function must disable preemption. : */ : static void rcu_qs(void) : { : RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n"); : if (__this_cpu_read(rcu_data.cpu_no_qs.s)) { 15.75 : ffff80001015df08: adrp x0, ffff800011529000 0.00 : ffff80001015df0c: add x0, x0, #0xac0 0.00 : ffff80001015df10: add x0, x0, #0x10 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001015df14: mrs x1, tpidr_el1 : rcu_qs(): 0.00 : ffff80001015df18: ldrh w2, [x0, x1] 0.00 : ffff80001015df1c: cbz w2, ffff80001015df2c : trace_rcu_grace_period(TPS("rcu_preempt"), : __this_cpu_read(rcu_data.gp_seq), : TPS("cpuqs")); : __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); 0.00 : ffff80001015df20: strb wzr, [x0, x1] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001015df24: mrs x0, sp_el0 : __write_once_size(): : } : : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff80001015df28: strb wzr, [x0, #781] : rcu_qs(): : barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */ : WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, false); : } : } 84.25 : ffff80001015df2c: ret Percent | Source code & Disassembly of vmlinux for cycles (39 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001045cd08 : : blk_finish_plug(): : * must be paired with an initial call to blk_start_plug(). The intent : * is to allow the block layer to optimize I/O submission. See the : * documentation for blk_start_plug() for more information. : */ : void blk_finish_plug(struct blk_plug *plug) : { 25.85 : ffff80001045cd08: stp x29, x30, [sp, #-32]! 0.00 : ffff80001045cd0c: mov x29, sp 40.85 : ffff80001045cd10: str x19, [sp, #16] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 5.08 : ffff80001045cd14: mrs x19, sp_el0 : blk_finish_plug(): : if (plug != current->plug) 0.00 : ffff80001045cd18: ldr x1, [x19, #1840] 0.00 : ffff80001045cd1c: cmp x1, x0 0.00 : ffff80001045cd20: b.eq ffff80001045cd30 // b.none : return; : blk_flush_plug_list(plug, false); : : current->plug = NULL; : } 0.00 : ffff80001045cd24: ldr x19, [sp, #16] 0.00 : ffff80001045cd28: ldp x29, x30, [sp], #32 0.00 : ffff80001045cd2c: ret : blk_flush_plug_list(plug, false); 15.32 : ffff80001045cd30: mov w1, #0x0 // #0 0.00 : ffff80001045cd34: bl ffff80001045cc00 : current->plug = NULL; 0.00 : ffff80001045cd38: str xzr, [x19, #1840] : } 12.90 : ffff80001045cd3c: ldr x19, [sp, #16] 0.00 : ffff80001045cd40: ldp x29, x30, [sp], #32 0.00 : ffff80001045cd44: ret Percent | Source code & Disassembly of vmlinux for cycles (35 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104568f0 : : bio_associate_blkg_from_css(): : * request_queue of the @bio. This falls back to the queue's root_blkg if : * the association fails with the css. : */ : void bio_associate_blkg_from_css(struct bio *bio, : struct cgroup_subsys_state *css) : { 0.00 : ffff8000104568f0: stp x29, x30, [sp, #-48]! 0.00 : ffff8000104568f4: mov x29, sp 2.91 : ffff8000104568f8: stp x19, x20, [sp, #16] 0.00 : ffff8000104568fc: mov x20, x0 2.62 : ffff800010456900: str x21, [sp, #32] 0.00 : ffff800010456904: mov x19, x1 : struct request_queue *q = bio->bi_disk->queue; 8.68 : ffff800010456908: ldr x0, [x0, #8] 8.68 : ffff80001045690c: ldr x21, [x0, #1040] : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff800010456910: bl ffff80001015c1b0 <__rcu_read_lock> : bio_associate_blkg_from_css(): : struct blkcg_gq *blkg; : : rcu_read_lock(); : : if (!css || !css->parent) 2.96 : ffff800010456914: cbz x19, ffff80001045694c 2.95 : ffff800010456918: ldr x0, [x19, #232] 0.00 : ffff80001045691c: cbz x0, ffff80001045694c : blkg = q->root_blkg; : else : blkg = blkg_lookup_create(css_to_blkcg(css), q); 59.63 : ffff800010456920: mov x1, x21 0.00 : ffff800010456924: mov x0, x19 0.00 : ffff800010456928: bl ffff80001047c6d0 0.00 : ffff80001045692c: mov x1, x0 : : __bio_associate_blkg(bio, blkg); 0.00 : ffff800010456930: add x0, x20, #0x48 0.00 : ffff800010456934: bl ffff800010456648 <__bio_associate_blkg.isra.40> : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff800010456938: bl ffff800010160d20 <__rcu_read_unlock> : bio_associate_blkg_from_css(): : : rcu_read_unlock(); : } 5.80 : ffff80001045693c: ldr x21, [sp, #32] 5.76 : ffff800010456940: ldp x19, x20, [sp, #16] 0.00 : ffff800010456944: ldp x29, x30, [sp], #48 0.00 : ffff800010456948: ret : blkg = q->root_blkg; 0.00 : ffff80001045694c: ldr x1, [x21, #1016] : __bio_associate_blkg(bio, blkg); 0.00 : ffff800010456950: add x0, x20, #0x48 0.00 : ffff800010456954: bl ffff800010456648 <__bio_associate_blkg.isra.40> : rcu_read_unlock(): 0.00 : ffff800010456958: bl ffff800010160d20 <__rcu_read_unlock> : bio_associate_blkg_from_css(): : } 0.00 : ffff80001045695c: ldr x21, [sp, #32] 0.00 : ffff800010456960: ldp x19, x20, [sp, #16] 0.00 : ffff800010456964: ldp x29, x30, [sp], #48 0.00 : ffff800010456968: ret Percent | Source code & Disassembly of vmlinux for cycles (43 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101f2c38 : : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 2.33 : ffff8000101f2c38: ldr x1, [x0, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff8000101f2c3c: sub x2, x1, #0x1 7.05 : ffff8000101f2c40: tst x1, #0x1 0.00 : ffff8000101f2c44: csel x0, x2, x0, ne // ne = any : __read_once_size(): 27.01 : ffff8000101f2c48: ldr x2, [x0, #8] : compound_head(): 0.00 : ffff8000101f2c4c: sub x1, x2, #0x1 0.00 : ffff8000101f2c50: tst x2, #0x1 0.00 : ffff8000101f2c54: csel x1, x1, x0, ne // ne = any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 16.50 : ffff8000101f2c58: ldr x1, [x1] : page_mapping(): : struct address_space *mapping; : : page = compound_head(page); : : /* This happens if someone calls flush_dcache_page on slab page */ : if (unlikely(PageSlab(page))) 0.00 : ffff8000101f2c5c: tst w1, #0x200 0.00 : ffff8000101f2c60: b.ne ffff8000101f2cc4 // b.any : __read_once_size(): 28.35 : ffff8000101f2c64: ldr x2, [x0, #8] : compound_head(): 0.00 : ffff8000101f2c68: sub x1, x2, #0x1 0.00 : ffff8000101f2c6c: tst x2, #0x1 0.00 : ffff8000101f2c70: csel x1, x1, x0, ne // ne = any : test_bit(): 2.23 : ffff8000101f2c74: ldr x1, [x1] : PageSwapCache(): : static __always_inline int PageSwapCache(struct page *page) : { : #ifdef CONFIG_THP_SWAP : page = compound_head(page); : #endif : return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags); 0.00 : ffff8000101f2c78: tst w1, #0x80000 0.00 : ffff8000101f2c7c: b.eq ffff8000101f2cb0 // b.none : test_bit(): 2.36 : ffff8000101f2c80: ldr x1, [x0] : PageSwapCache(): 0.00 : ffff8000101f2c84: tst w1, #0x400 0.00 : ffff8000101f2c88: b.eq ffff8000101f2cb0 // b.none : page_mapping(): : return NULL; : : if (unlikely(PageSwapCache(page))) { : swp_entry_t entry; : : entry.val = page_private(page); 0.00 : ffff8000101f2c8c: ldr x0, [x0, #40] : return swap_address_space(entry); 0.00 : ffff8000101f2c90: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101f2c94: add x1, x1, #0x4e8 0.00 : ffff8000101f2c98: lsr x2, x0, #58 0.00 : ffff8000101f2c9c: ubfx x0, x0, #14, #44 0.00 : ffff8000101f2ca0: add x0, x0, x0, lsl #2 0.00 : ffff8000101f2ca4: ldr x1, [x1, x2, lsl #3] 0.00 : ffff8000101f2ca8: add x0, x1, x0, lsl #5 : mapping = page->mapping; : if ((unsigned long)mapping & PAGE_MAPPING_ANON) : return NULL; : : return (void *)((unsigned long)mapping & ~PAGE_MAPPING_FLAGS); : } 0.00 : ffff8000101f2cac: ret : if ((unsigned long)mapping & PAGE_MAPPING_ANON) 2.42 : ffff8000101f2cb0: ldr x1, [x0, #24] : return (void *)((unsigned long)mapping & ~PAGE_MAPPING_FLAGS); 0.00 : ffff8000101f2cb4: and x0, x1, #0xfffffffffffffffc 0.00 : ffff8000101f2cb8: tst x1, #0x1 0.00 : ffff8000101f2cbc: csel x0, x0, xzr, eq // eq = none : } 11.75 : ffff8000101f2cc0: ret : return NULL; 0.00 : ffff8000101f2cc4: mov x0, #0x0 // #0 : } 0.00 : ffff8000101f2cc8: ret Percent | Source code & Disassembly of vmlinux for cycles (16 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cb93f0 : : cpumask_next_and(): : * : * Returns >= nr_cpu_ids if no further cpus set in both. : */ : int cpumask_next_and(int n, const struct cpumask *src1p, : const struct cpumask *src2p) : { 87.69 : ffff800010cb93f0: stp x29, x30, [sp, #-16]! : /* -1 is a legal arg here. */ : if (n != -1) : cpumask_check(n); : return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), : nr_cpumask_bits, n + 1); 0.00 : ffff800010cb93f4: add w3, w0, #0x1 : { 0.00 : ffff800010cb93f8: mov x0, x1 0.00 : ffff800010cb93fc: mov x1, x2 0.00 : ffff800010cb9400: mov x29, sp : return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), 0.00 : ffff800010cb9404: sxtw x3, w3 0.00 : ffff800010cb9408: mov x2, #0x100 // #256 0.00 : ffff800010cb940c: bl ffff80001048da10 : } 12.31 : ffff800010cb9410: ldp x29, x30, [sp], #16 0.00 : ffff800010cb9414: ret Percent | Source code & Disassembly of vmlinux for cycles (12 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101f75f8 : : quiet_vmstat(): : * until the diffs stay at zero. The function is used by NOHZ and can only be : * invoked when tick processing is not active. : */ : void quiet_vmstat(void) : { : if (system_state != SYSTEM_RUNNING) 16.62 : ffff8000101f75f8: adrp x0, ffff800011909000 0.00 : ffff8000101f75fc: ldr w0, [x0, #2304] 0.00 : ffff8000101f7600: cmp w0, #0x2 0.00 : ffff8000101f7604: b.eq ffff8000101f760c // b.none 0.00 : ffff8000101f7608: ret : { 0.00 : ffff8000101f760c: stp x29, x30, [sp, #-16]! : return; : : if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 0.00 : ffff8000101f7610: adrp x0, ffff800011526000 0.00 : ffff8000101f7614: add x0, x0, #0x9b8 : { 0.00 : ffff8000101f7618: mov x29, sp : if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 0.00 : ffff8000101f761c: add x0, x0, #0x2a0 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000101f7620: mrs x1, tpidr_el1 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000101f7624: ldr x0, [x0, x1] : quiet_vmstat(): 0.00 : ffff8000101f7628: tbnz w0, #0, ffff8000101f7634 : * vmstat_update. It doesn't fire that often to matter and canceling : * it would be too expensive from this path. : * vmstat_shepherd will take care about that for us. : */ : refresh_cpu_vm_stats(false); : } 83.38 : ffff8000101f762c: ldp x29, x30, [sp], #16 0.00 : ffff8000101f7630: ret : if (!need_update(smp_processor_id())) 0.00 : ffff8000101f7634: adrp x0, ffff80001151d000 0.00 : ffff8000101f7638: add x0, x0, #0x18 0.00 : ffff8000101f763c: ldr w0, [x0, x1] 0.00 : ffff8000101f7640: bl ffff8000101f3ff0 0.00 : ffff8000101f7644: tst w0, #0xff 0.00 : ffff8000101f7648: b.eq ffff8000101f762c // b.none : refresh_cpu_vm_stats(false); 0.00 : ffff8000101f764c: mov w0, #0x0 // #0 0.00 : ffff8000101f7650: bl ffff8000101f5088 0.00 : ffff8000101f7654: b ffff8000101f762c Percent | Source code & Disassembly of vmlinux for cycles (45 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001047c6d0 : : blkg_lookup_create(): : * This looks up or creates the blkg representing the unique pair : * of the blkcg and the request_queue. : */ : struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, : struct request_queue *q) : { 8.88 : ffff80001047c6d0: stp x29, x30, [sp, #-64]! : __blkg_lookup(): : struct request_queue *q, : bool update_hint) : { : struct blkcg_gq *blkg; : : if (blkcg == &blkcg_root) 0.00 : ffff80001047c6d4: adrp x2, ffff800011b3b000 0.00 : ffff80001047c6d8: add x2, x2, #0x100 : blkg_lookup_create(): 0.00 : ffff80001047c6dc: mov x29, sp 13.35 : ffff80001047c6e0: stp x19, x20, [sp, #16] : __blkg_lookup(): 0.00 : ffff80001047c6e4: add x2, x2, #0x28 : blkg_lookup_create(): 2.21 : ffff80001047c6e8: str x21, [sp, #32] 0.00 : ffff80001047c6ec: mov x20, x0 : __blkg_lookup(): 0.00 : ffff80001047c6f0: cmp x0, x2 : blkg_lookup_create(): 0.00 : ffff80001047c6f4: mov x21, x1 : __blkg_lookup(): 0.00 : ffff80001047c6f8: b.eq ffff80001047c73c // b.none : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001047c6fc: ldr x19, [x0, #264] : __blkg_lookup(): : return q->root_blkg; : : blkg = rcu_dereference(blkcg->blkg_hint); : if (blkg && blkg->q == q) 4.48 : ffff80001047c700: cbz x19, ffff80001047c710 2.23 : ffff80001047c704: ldr x0, [x19] 0.00 : ffff80001047c708: cmp x1, x0 0.00 : ffff80001047c70c: b.eq ffff80001047c728 // b.none : return blkg; : : return blkg_lookup_slowpath(blkcg, q, update_hint); 0.00 : ffff80001047c710: mov w2, #0x0 // #0 0.00 : ffff80001047c714: mov x1, x21 0.00 : ffff80001047c718: mov x0, x20 0.00 : ffff80001047c71c: bl ffff80001047a550 0.00 : ffff80001047c720: mov x19, x0 : blkg_lookup_create(): : struct blkcg_gq *blkg = blkg_lookup(blkcg, q); : : if (unlikely(!blkg)) { 0.00 : ffff80001047c724: cbz x19, ffff80001047c744 : blkg = __blkg_lookup_create(blkcg, q); : spin_unlock_irqrestore(&q->queue_lock, flags); : } : : return blkg; : } 62.18 : ffff80001047c728: mov x0, x19 6.68 : ffff80001047c72c: ldr x21, [sp, #32] 0.00 : ffff80001047c730: ldp x19, x20, [sp, #16] 0.00 : ffff80001047c734: ldp x29, x30, [sp], #64 0.00 : ffff80001047c738: ret : __blkg_lookup(): : return q->root_blkg; 0.00 : ffff80001047c73c: ldr x19, [x1, #1016] : blkg_lookup_create(): : if (unlikely(!blkg)) { 0.00 : ffff80001047c740: cbnz x19, ffff80001047c728 0.00 : ffff80001047c744: stp x22, x23, [x29, #40] : spinlock_check(): : * Map the spin_lock functions to the raw variants for PREEMPT_RT=n : */ : : static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock) : { : return &lock->rlock; 0.00 : ffff80001047c748: add x22, x21, #0x7c : blkg_lookup_create(): : spin_lock_irqsave(&q->queue_lock, flags); 0.00 : ffff80001047c74c: mov x0, x22 0.00 : ffff80001047c750: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : blkg = __blkg_lookup_create(blkcg, q); 0.00 : ffff80001047c754: mov x1, x21 : spin_lock_irqsave(&q->queue_lock, flags); 0.00 : ffff80001047c758: mov x23, x0 : blkg = __blkg_lookup_create(blkcg, q); 0.00 : ffff80001047c75c: mov x0, x20 0.00 : ffff80001047c760: bl ffff80001047c560 <__blkg_lookup_create> : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff80001047c764: mov x1, x23 : blkg_lookup_create(): 0.00 : ffff80001047c768: mov x19, x0 : spin_unlock_irqrestore(): 0.00 : ffff80001047c76c: mov x0, x22 0.00 : ffff80001047c770: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : blkg_lookup_create(): : } 0.00 : ffff80001047c774: mov x0, x19 0.00 : ffff80001047c778: ldr x21, [sp, #32] : spin_unlock_irqrestore(): 0.00 : ffff80001047c77c: ldp x22, x23, [x29, #40] : blkg_lookup_create(): 0.00 : ffff80001047c780: ldp x19, x20, [sp, #16] 0.00 : ffff80001047c784: ldp x29, x30, [sp], #64 0.00 : ffff80001047c788: ret Percent | Source code & Disassembly of vmlinux for cycles (12 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011bdd0 : : put_prev_task_idle(): : resched_curr(rq); : } : : static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) : { : } 100.00 : ffff80001011bdd0: ret Percent | Source code & Disassembly of vmlinux for cycles (14 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101f3ff0 : : need_update(): : /* : * Check if the diffs for a certain cpu indicate that : * an update is needed. : */ : static bool need_update(int cpu) : { 0.00 : ffff8000101f3ff0: stp x29, x30, [sp, #-48]! 0.00 : ffff8000101f3ff4: mov x29, sp 7.45 : ffff8000101f3ff8: str x21, [sp, #32] 0.00 : ffff8000101f3ffc: sxtw x21, w0 : struct zone *zone; : : for_each_populated_zone(zone) { 0.00 : ffff8000101f4000: bl ffff8000101f3c20 0.00 : ffff8000101f4004: cbz x0, ffff8000101f4074 0.00 : ffff8000101f4008: stp x19, x20, [x29, #16] 0.00 : ffff8000101f400c: mov x19, x0 0.00 : ffff8000101f4010: str x22, [x29, #40] : struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu); 0.00 : ffff8000101f4014: adrp x22, ffff800011909000 0.00 : ffff8000101f4018: add x22, x22, #0x928 0.00 : ffff8000101f401c: nop : for_each_populated_zone(zone) { 7.48 : ffff8000101f4020: ldr x3, [x19, #120] : #endif : : /* : * The fast way of checking if there are any vmstat diffs. : */ : if (memchr_inv(p->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS * 0.00 : ffff8000101f4024: mov x2, #0xc // #12 0.00 : ffff8000101f4028: mov w1, #0x0 // #0 : for_each_populated_zone(zone) { 0.00 : ffff8000101f402c: cbz x3, ffff8000101f405c : struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu); 21.11 : ffff8000101f4030: ldr x20, [x22, x21, lsl #3] 7.16 : ffff8000101f4034: ldr x3, [x19, #88] 0.00 : ffff8000101f4038: add x20, x3, x20 : if (memchr_inv(p->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS * 0.00 : ffff8000101f403c: add x0, x20, #0x4f 0.00 : ffff8000101f4040: bl ffff800010cc82b0 0.00 : ffff8000101f4044: cbnz x0, ffff8000101f4084 : sizeof(p->vm_stat_diff[0]))) : return true; : #ifdef CONFIG_NUMA : if (memchr_inv(p->vm_numa_stat_diff, 0, NR_VM_NUMA_STAT_ITEMS * 0.00 : ffff8000101f4048: mov x2, #0xc // #12 0.00 : ffff8000101f404c: mov w1, #0x0 // #0 0.00 : ffff8000101f4050: add x0, x20, #0x42 0.00 : ffff8000101f4054: bl ffff800010cc82b0 0.00 : ffff8000101f4058: cbnz x0, ffff8000101f4084 : for_each_populated_zone(zone) { 36.21 : ffff8000101f405c: mov x0, x19 0.00 : ffff8000101f4060: bl ffff8000101f3cb8 20.59 : ffff8000101f4064: mov x19, x0 0.00 : ffff8000101f4068: cbnz x0, ffff8000101f4020 0.00 : ffff8000101f406c: ldp x19, x20, [x29, #16] 0.00 : ffff8000101f4070: ldr x22, [x29, #40] : sizeof(p->vm_numa_stat_diff[0]))) : return true; : #endif : } : return false; 0.00 : ffff8000101f4074: mov w0, #0x0 // #0 : } 0.00 : ffff8000101f4078: ldr x21, [sp, #32] 0.00 : ffff8000101f407c: ldp x29, x30, [sp], #48 0.00 : ffff8000101f4080: ret : return true; 0.00 : ffff8000101f4084: mov w0, #0x1 // #1 0.00 : ffff8000101f4088: ldr x22, [x29, #40] 0.00 : ffff8000101f408c: ldp x19, x20, [x29, #16] : } 0.00 : ffff8000101f4090: ldr x21, [sp, #32] 0.00 : ffff8000101f4094: ldp x29, x30, [sp], #48 0.00 : ffff8000101f4098: ret Percent | Source code & Disassembly of vmlinux for cycles (11 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016bf00 : : hrtimer_run_queues(): : : /* : * Called from run_local_timers in hardirq context every jiffy : */ : void hrtimer_run_queues(void) : { 0.00 : ffff80001016bf00: stp x29, x30, [sp, #-48]! 0.00 : ffff80001016bf04: mov x29, sp 8.78 : ffff80001016bf08: stp x19, x20, [sp, #16] : struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016bf0c: adrp x19, ffff800011525000 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 8.99 : ffff80001016bf10: mrs x20, tpidr_el1 : hrtimer_run_queues(): 0.00 : ffff80001016bf14: add x19, x19, #0x980 0.00 : ffff80001016bf18: add x20, x19, x20 : __hrtimer_hres_active(): : cpu_base->hres_active : 0; 0.00 : ffff80001016bf1c: ldrb w0, [x20, #16] : hrtimer_run_queues(): : unsigned long flags; : ktime_t now; : : if (__hrtimer_hres_active(cpu_base)) 0.00 : ffff80001016bf20: tbz w0, #0, ffff80001016bf30 : raise_softirq_irqoff(HRTIMER_SOFTIRQ); : } : : __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); : } 73.27 : ffff80001016bf24: ldp x19, x20, [sp, #16] 8.97 : ffff80001016bf28: ldp x29, x30, [sp], #48 0.00 : ffff80001016bf2c: ret 0.00 : ffff80001016bf30: str x21, [x29, #32] : hrtimer_is_hres_enabled(): : return hrtimer_hres_enabled; 0.00 : ffff80001016bf34: adrp x21, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001016bf38: add x0, x21, #0x2bc : hrtimer_run_queues(): : if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) { 0.00 : ffff80001016bf3c: ldrb w0, [x0, #4] 0.00 : ffff80001016bf40: eor w0, w0, #0x1 0.00 : ffff80001016bf44: bl ffff80001017c4a0 0.00 : ffff80001016bf48: cbnz w0, ffff80001016bfc4 : raw_spin_lock_irqsave(&cpu_base->lock, flags); 0.00 : ffff80001016bf4c: mov x0, x20 0.00 : ffff80001016bf50: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : hrtimer_update_base(): : ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 0.00 : ffff80001016bf54: add x3, x20, #0x138 0.00 : ffff80001016bf58: add x2, x20, #0xf8 0.00 : ffff80001016bf5c: add x1, x20, #0xb8 : hrtimer_run_queues(): : raw_spin_lock_irqsave(&cpu_base->lock, flags); 0.00 : ffff80001016bf60: mov x19, x0 : hrtimer_update_base(): : ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 0.00 : ffff80001016bf64: add x0, x20, #0xc 0.00 : ffff80001016bf68: bl ffff80001016ea30 : base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; 0.00 : ffff80001016bf6c: ldr x3, [x20, #184] : ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, 0.00 : ffff80001016bf70: mov x21, x0 : base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; 0.00 : ffff80001016bf74: ldr x2, [x20, #248] : base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; 0.00 : ffff80001016bf78: ldr x1, [x20, #312] : ktime_compare(): : * cmp1 == cmp2: return 0 : * cmp1 > cmp2: return >0 : */ : static inline int ktime_compare(const ktime_t cmp1, const ktime_t cmp2) : { : if (cmp1 < cmp2) 0.00 : ffff80001016bf7c: ldr x0, [x20, #48] : hrtimer_update_base(): : base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; 0.00 : ffff80001016bf80: str x3, [x20, #440] : base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; 0.00 : ffff80001016bf84: str x2, [x20, #504] : ktime_compare(): 0.00 : ffff80001016bf88: cmp x0, x21 : hrtimer_update_base(): : base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; 0.00 : ffff80001016bf8c: str x1, [x20, #568] : ktime_compare(): 0.00 : ffff80001016bf90: b.le ffff80001016c004 : hrtimer_run_queues(): : __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); 0.00 : ffff80001016bf94: mov x2, x19 0.00 : ffff80001016bf98: mov w3, #0xf // #15 0.00 : ffff80001016bf9c: mov x1, x21 0.00 : ffff80001016bfa0: mov x0, x20 0.00 : ffff80001016bfa4: bl ffff80001016b278 <__hrtimer_run_queues> : raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 0.00 : ffff80001016bfa8: mov x1, x19 0.00 : ffff80001016bfac: mov x0, x20 0.00 : ffff80001016bfb0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff80001016bfb4: ldr x21, [x29, #32] : } 0.00 : ffff80001016bfb8: ldp x19, x20, [sp, #16] 0.00 : ffff80001016bfbc: ldp x29, x30, [sp], #48 0.00 : ffff80001016bfc0: ret : __my_cpu_offset(): 0.00 : ffff80001016bfc4: mrs x0, tpidr_el1 : hrtimer_switch_to_hres(): : struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases); 0.00 : ffff80001016bfc8: add x19, x19, x0 : if (tick_init_highres()) { 0.00 : ffff80001016bfcc: bl ffff80001017b1d0 0.00 : ffff80001016bfd0: cbnz w0, ffff80001016c024 : base->hres_active = 1; 0.00 : ffff80001016bfd4: ldrb w0, [x19, #16] : hrtimer_resolution = HIGH_RES_NSEC; 0.00 : ffff80001016bfd8: mov w1, #0x1 // #1 0.00 : ffff80001016bfdc: str w1, [x21, #700] : base->hres_active = 1; 0.00 : ffff80001016bfe0: orr w0, w0, w1 0.00 : ffff80001016bfe4: strb w0, [x19, #16] : tick_setup_sched_timer(); 0.00 : ffff80001016bfe8: bl ffff80001017c1e8 : retrigger_next_event(NULL); 0.00 : ffff80001016bfec: mov x0, #0x0 // #0 0.00 : ffff80001016bff0: bl ffff80001016b430 0.00 : ffff80001016bff4: ldr x21, [x29, #32] : hrtimer_run_queues(): : } 0.00 : ffff80001016bff8: ldp x19, x20, [sp, #16] 0.00 : ffff80001016bffc: ldp x29, x30, [sp], #48 0.00 : ffff80001016c000: ret : cpu_base->softirq_activated = 1; 0.00 : ffff80001016c004: ldrb w1, [x20, #16] : cpu_base->softirq_expires_next = KTIME_MAX; 0.00 : ffff80001016c008: mov x0, #0x7fffffffffffffff // #9223372036854775807 0.00 : ffff80001016c00c: str x0, [x20, #48] : raise_softirq_irqoff(HRTIMER_SOFTIRQ); 0.00 : ffff80001016c010: mov w0, #0x8 // #8 : cpu_base->softirq_activated = 1; 0.00 : ffff80001016c014: orr w1, w1, w0 0.00 : ffff80001016c018: strb w1, [x20, #16] : raise_softirq_irqoff(HRTIMER_SOFTIRQ); 0.00 : ffff80001016c01c: bl ffff8000100ecd98 0.00 : ffff80001016c020: b ffff80001016bf94 : hrtimer_switch_to_hres(): : pr_warn("Could not switch to high resolution mode on CPU %u\n", 0.00 : ffff80001016c024: ldr w1, [x19, #4] 0.00 : ffff80001016c028: adrp x0, ffff8000111c8000 0.00 : ffff80001016c02c: add x0, x0, #0xad0 0.00 : ffff80001016c030: bl ffff80001014a85c 0.00 : ffff80001016c034: ldr x21, [x29, #32] 0.00 : ffff80001016c038: b ffff80001016bf24 Percent | Source code & Disassembly of vmlinux for cycles (23 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104884f8 : : import_single_range(): : EXPORT_SYMBOL(compat_import_iovec); : #endif : : int import_single_range(int rw, void __user *buf, size_t len, : struct iovec *iov, struct iov_iter *i) : { 13.25 : ffff8000104884f8: stp x29, x30, [sp, #-16]! 0.00 : ffff8000104884fc: mov x6, #0x7ffff000 // #2147479552 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 13.26 : ffff800010488500: mrs x5, sp_el0 : import_single_range(): 0.00 : ffff800010488504: mov x29, sp : __range_ok(): : * Asynchronous I/O running in a kernel thread does not have the : * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag : * the user address before checking. : */ : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && : (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR))) 0.00 : ffff800010488508: ldr w8, [x5, #44] 0.00 : ffff80001048850c: cmp x2, x6 0.00 : ffff800010488510: csel x6, x2, x6, ls // ls = plast : unsigned long ret, limit = current_thread_info()->addr_limit; 29.64 : ffff800010488514: ldr x7, [x5, #8] : if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) && 0.00 : ffff800010488518: tbnz w8, #21, ffff80001048852c : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001048851c: ldr x2, [x5] : __range_ok(): 0.00 : ffff800010488520: mov x5, x1 0.00 : ffff800010488524: tst w2, #0x4000000 0.00 : ffff800010488528: b.eq ffff800010488534 // b.none : sign_extend64(): : * @index: 0 based bit index (0<=index<64) to sign bit : */ : static inline __s64 sign_extend64(__u64 value, int index) : { : __u8 shift = 63 - index; : return (__s64)(value << shift) >> shift; 0.00 : ffff80001048852c: sbfx x5, x1, #0, #56 : __range_ok(): : addr = untagged_addr(addr); 0.00 : ffff800010488530: and x5, x1, x5 : : __chk_user_ptr(addr); : asm volatile( 4.40 : ffff800010488534: adds x5, x5, x6 0.00 : ffff800010488538: csel x7, xzr, x7, hi // hi = pmore 0.00 : ffff80001048853c: csinv x5, x5, xzr, cc // cc = lo, ul, last 4.36 : ffff800010488540: sbcs xzr, x5, x7 4.31 : ffff800010488544: cset x5, ls // ls = plast : import_single_range(): : if (len > MAX_RW_COUNT) : len = MAX_RW_COUNT; : if (unlikely(!access_ok(buf, len))) 0.00 : ffff800010488548: cbz x5, ffff80001048857c 0.00 : ffff80001048854c: mov x2, x3 0.00 : ffff800010488550: mov x7, x4 0.00 : ffff800010488554: mov x5, x1 : return -EFAULT; : : iov->iov_base = buf; : iov->iov_len = len; : iov_iter_init(i, rw, iov, 1, len); 0.00 : ffff800010488558: mov x4, x6 26.38 : ffff80001048855c: mov w1, w0 0.00 : ffff800010488560: mov x3, #0x1 // #1 : iov->iov_len = len; 0.00 : ffff800010488564: stp x5, x6, [x2] : iov_iter_init(i, rw, iov, 1, len); 0.00 : ffff800010488568: mov x0, x7 0.00 : ffff80001048856c: bl ffff800010487e48 : return 0; 4.40 : ffff800010488570: mov w0, #0x0 // #0 : } 0.00 : ffff800010488574: ldp x29, x30, [sp], #16 0.00 : ffff800010488578: ret : return -EFAULT; 0.00 : ffff80001048857c: mov w0, #0xfffffff2 // #-14 : } 0.00 : ffff800010488580: ldp x29, x30, [sp], #16 0.00 : ffff800010488584: ret Percent | Source code & Disassembly of vmlinux for cycles (32 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010489108 : : iov_iter_alignment(): : i->iov_offset = 0; : } : EXPORT_SYMBOL(iov_iter_discard); : : unsigned long iov_iter_alignment(const struct iov_iter *i) : { 25.16 : ffff800010489108: stp x29, x30, [sp, #-32]! 0.00 : ffff80001048910c: mov x29, sp 0.00 : ffff800010489110: str x19, [sp, #16] : iov_iter_is_pipe(): : return iov_iter_type(i) == ITER_BVEC; : } : : static inline bool iov_iter_is_pipe(const struct iov_iter *i) : { : return iov_iter_type(i) == ITER_PIPE; 0.00 : ffff800010489114: ldr w3, [x0] : iov_iter_alignment(): : unsigned long res = 0; : size_t size = i->count; 3.12 : ffff800010489118: ldr x2, [x0, #16] : iov_iter_type(): : return i->type & ~(READ | WRITE); 0.00 : ffff80001048911c: and w1, w3, #0xfffffffe : iov_iter_alignment(): : : if (unlikely(iov_iter_is_pipe(i))) { 0.00 : ffff800010489120: cmp w1, #0x20 0.00 : ffff800010489124: b.eq ffff8000104891bc // b.none : : if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) : return size | i->iov_offset; : return size; : } : iterate_all_kinds(i, size, v, 3.18 : ffff800010489128: cbz x2, ffff8000104891a8 0.00 : ffff80001048912c: ldr x1, [x0, #8] 0.00 : ffff800010489130: tbnz w3, #4, ffff80001048920c 0.00 : ffff800010489134: tbnz w3, #3, ffff80001048929c 0.00 : ffff800010489138: tbnz w3, #6, ffff8000104891a8 3.15 : ffff80001048913c: ldr x3, [x0, #24] 3.15 : ffff800010489140: ldr x0, [x3, #8] 0.00 : ffff800010489144: sub x0, x0, x1 0.00 : ffff800010489148: cmp x0, x2 0.00 : ffff80001048914c: csel x0, x0, x2, ls // ls = plast 15.48 : ffff800010489150: cbz x0, ffff800010489178 0.00 : ffff800010489154: ldr x19, [x3] 0.00 : ffff800010489158: subs x2, x2, x0 0.00 : ffff80001048915c: add x1, x19, x1 0.00 : ffff800010489160: orr x19, x1, x0 40.47 : ffff800010489164: b.ne ffff80001048917c // b.any : (res |= (unsigned long)v.iov_base | v.iov_len, 0), : res |= v.bv_offset | v.bv_len, : res |= (unsigned long)v.iov_base | v.iov_len : ) : return res; : } 0.00 : ffff800010489168: mov x0, x19 6.29 : ffff80001048916c: ldr x19, [sp, #16] 0.00 : ffff800010489170: ldp x29, x30, [sp], #32 0.00 : ffff800010489174: ret : unsigned long res = 0; 0.00 : ffff800010489178: mov x19, #0x0 // #0 : iterate_all_kinds(i, size, v, 0.00 : ffff80001048917c: add x3, x3, #0x10 0.00 : ffff800010489180: ldr x1, [x3, #8] 0.00 : ffff800010489184: cmp x1, x2 0.00 : ffff800010489188: csel x1, x1, x2, ls // ls = plast 0.00 : ffff80001048918c: cbz x1, ffff80001048917c 0.00 : ffff800010489190: ldr x0, [x3] 0.00 : ffff800010489194: subs x2, x2, x1 0.00 : ffff800010489198: orr x0, x0, x19 0.00 : ffff80001048919c: orr x19, x1, x0 0.00 : ffff8000104891a0: b.eq ffff800010489168 // b.none 0.00 : ffff8000104891a4: b ffff80001048917c 0.00 : ffff8000104891a8: mov x19, #0x0 // #0 : } 0.00 : ffff8000104891ac: mov x0, x19 0.00 : ffff8000104891b0: ldr x19, [sp, #16] 0.00 : ffff8000104891b4: ldp x29, x30, [sp], #32 0.00 : ffff8000104891b8: ret 0.00 : ffff8000104891bc: mov x19, #0x0 // #0 : if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) 0.00 : ffff8000104891c0: cbz x2, ffff800010489168 0.00 : ffff8000104891c4: ldr x1, [x0, #8] 0.00 : ffff8000104891c8: mov x19, x2 0.00 : ffff8000104891cc: cbz x1, ffff800010489168 : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff8000104891d0: ldr x3, [x0, #24] : if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) 0.00 : ffff8000104891d4: mov w4, #0x28 // #40 0.00 : ffff8000104891d8: ldr w5, [x0, #32] 0.00 : ffff8000104891dc: adrp x2, ffff800010d18000 0.00 : ffff8000104891e0: add x2, x2, #0xf80 : return size | i->iov_offset; 0.00 : ffff8000104891e4: orr x1, x1, x19 : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff8000104891e8: ldr w0, [x3, #92] : if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) 0.00 : ffff8000104891ec: ldr x3, [x3, #144] : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff8000104891f0: sub w0, w0, #0x1 : if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) 0.00 : ffff8000104891f4: and w0, w0, w5 0.00 : ffff8000104891f8: umaddl x0, w0, w4, x3 0.00 : ffff8000104891fc: ldr x0, [x0, #16] : return size | i->iov_offset; 0.00 : ffff800010489200: cmp x0, x2 0.00 : ffff800010489204: csel x19, x1, x19, eq // eq = none 0.00 : ffff800010489208: b ffff800010489168 : iterate_all_kinds(i, size, v, 0.00 : ffff80001048920c: mov w3, w2 0.00 : ffff800010489210: mov w4, #0x0 // #0 : unsigned long res = 0; 0.00 : ffff800010489214: mov x19, #0x0 // #0 : iterate_all_kinds(i, size, v, 0.00 : ffff800010489218: mov w9, #0x1000 // #4096 0.00 : ffff80001048921c: nop 0.00 : ffff800010489220: cbz w3, ffff800010489168 0.00 : ffff800010489224: ldr x7, [x0, #24] 0.00 : ffff800010489228: ubfiz x5, x4, #4, #32 0.00 : ffff80001048922c: add x5, x7, x5 0.00 : ffff800010489230: ldp w2, w6, [x5, #8] 0.00 : ffff800010489234: sub w5, w2, w1 0.00 : ffff800010489238: add w6, w1, w6 0.00 : ffff80001048923c: cmp w5, w3 0.00 : ffff800010489240: and w6, w6, #0xfff 0.00 : ffff800010489244: csel w5, w5, w3, ls // ls = plast 0.00 : ffff800010489248: sub w8, w9, w6 0.00 : ffff80001048924c: cmp w5, w8 0.00 : ffff800010489250: csel w5, w5, w8, ls // ls = plast 0.00 : ffff800010489254: cbz w5, ffff800010489268 0.00 : ffff800010489258: orr w6, w6, w5 : bvec_iter_advance(): : static inline bool bvec_iter_advance(const struct bio_vec *bv, : struct bvec_iter *iter, unsigned bytes) : { : unsigned int idx = iter->bi_idx; : : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff80001048925c: cmp w3, w5 : iov_iter_alignment(): 0.00 : ffff800010489260: orr x19, x19, x6 : bvec_iter_advance(): 0.00 : ffff800010489264: b.cc ffff8000104892f4 // b.lo, b.ul, b.last : "Attempted to advance past end of bvec iter\n")) { : iter->bi_size = 0; : return false; : } : : iter->bi_size -= bytes; 0.00 : ffff800010489268: sub w3, w3, w5 : bytes += iter->bi_bvec_done; : : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff80001048926c: adds w1, w1, w5 0.00 : ffff800010489270: b.eq ffff800010489220 // b.none 0.00 : ffff800010489274: nop 0.00 : ffff800010489278: cmp w2, w1 0.00 : ffff80001048927c: b.hi ffff800010489220 // b.pmore 0.00 : ffff800010489280: subs w1, w1, w2 : bytes -= bv[idx].bv_len; : idx++; 0.00 : ffff800010489284: add w4, w4, #0x1 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010489288: b.eq ffff800010489220 // b.none 0.00 : ffff80001048928c: ubfiz x2, x4, #4, #32 0.00 : ffff800010489290: add x2, x7, x2 0.00 : ffff800010489294: ldr w2, [x2, #8] 0.00 : ffff800010489298: b ffff800010489278 : iov_iter_alignment(): 0.00 : ffff80001048929c: ldr x3, [x0, #24] 0.00 : ffff8000104892a0: ldr x0, [x3, #8] 0.00 : ffff8000104892a4: sub x0, x0, x1 0.00 : ffff8000104892a8: cmp x0, x2 0.00 : ffff8000104892ac: csel x0, x0, x2, ls // ls = plast 0.00 : ffff8000104892b0: cbz x0, ffff80001048931c 0.00 : ffff8000104892b4: ldr x19, [x3] 0.00 : ffff8000104892b8: subs x2, x2, x0 0.00 : ffff8000104892bc: add x1, x19, x1 0.00 : ffff8000104892c0: orr x19, x1, x0 0.00 : ffff8000104892c4: b.eq ffff800010489168 // b.none 0.00 : ffff8000104892c8: add x3, x3, #0x10 0.00 : ffff8000104892cc: ldr x0, [x3, #8] 0.00 : ffff8000104892d0: cmp x0, x2 0.00 : ffff8000104892d4: csel x0, x0, x2, ls // ls = plast 0.00 : ffff8000104892d8: cbz x0, ffff8000104892c8 0.00 : ffff8000104892dc: ldr x1, [x3] 0.00 : ffff8000104892e0: subs x2, x2, x0 0.00 : ffff8000104892e4: orr x19, x1, x19 0.00 : ffff8000104892e8: orr x19, x19, x0 0.00 : ffff8000104892ec: b.eq ffff800010489168 // b.none 0.00 : ffff8000104892f0: b ffff8000104892c8 : bvec_iter_advance(): : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff8000104892f4: adrp x1, ffff800011ad8000 0.00 : ffff8000104892f8: ldrb w0, [x1, #1302] 0.00 : ffff8000104892fc: cbnz w0, ffff800010489168 0.00 : ffff800010489300: mov w2, #0x1 // #1 0.00 : ffff800010489304: adrp x0, ffff8000111e9000 0.00 : ffff800010489308: strb w2, [x1, #1302] 0.00 : ffff80001048930c: add x0, x0, #0x680 0.00 : ffff800010489310: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010489314: brk #0x800 0.00 : ffff800010489318: b ffff800010489168 : iov_iter_alignment(): : unsigned long res = 0; 0.00 : ffff80001048931c: mov x19, #0x0 // #0 0.00 : ffff800010489320: b ffff8000104892c8 Percent | Source code & Disassembly of vmlinux for cycles (34 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010455378 <__bio_try_merge_page>: : __bio_try_merge_page(): : * : * Return %true on success or %false on failure. : */ : bool __bio_try_merge_page(struct bio *bio, struct page *page, : unsigned int len, unsigned int off, bool *same_page) : { 0.00 : ffff800010455378: stp x29, x30, [sp, #-80]! 0.00 : ffff80001045537c: mov x29, sp : bio_flagged(): : atomic_set(&bio->__bi_cnt, count); : } : : static inline bool bio_flagged(struct bio *bio, unsigned int bit) : { : return (bio->bi_flags & (1U << bit)) != 0; 0.00 : ffff800010455380: ldrh w5, [x0, #20] : __bio_try_merge_page(): : if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) 0.00 : ffff800010455384: tbnz w5, #1, ffff8000104554a8 <__bio_try_merge_page+0x130> : return false; : : if (bio->bi_vcnt > 0) { 0.00 : ffff800010455388: ldrh w5, [x0, #96] 0.00 : ffff80001045538c: cbnz w5, ffff80001045539c <__bio_try_merge_page+0x24> : bv->bv_len += len; : bio->bi_iter.bi_size += len; : return true; : } : } : return false; 2.98 : ffff800010455390: mov w0, #0x0 // #0 : } 97.02 : ffff800010455394: ldp x29, x30, [sp], #80 0.00 : ffff800010455398: ret 0.00 : ffff80001045539c: stp x19, x20, [x29, #16] : struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 0.00 : ffff8000104553a0: mov x7, #0xfffffffffffffff0 // #-16 0.00 : ffff8000104553a4: str x21, [x29, #32] 0.00 : ffff8000104553a8: add x5, x7, w5, uxtw #4 0.00 : ffff8000104553ac: stp x24, x25, [x29, #56] : page_is_mergeable(): : phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + 0.00 : ffff8000104553b0: adrp x25, ffff8000112f2000 0.00 : ffff8000104553b4: mov x21, x1 : __bio_try_merge_page(): : struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 0.00 : ffff8000104553b8: ldr x8, [x0, #104] : page_is_mergeable(): : phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + 0.00 : ffff8000104553bc: ldr x7, [x25, #1880] : __bio_try_merge_page(): : struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 0.00 : ffff8000104553c0: add x24, x8, x5 : page_is_mergeable(): : phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + 0.00 : ffff8000104553c4: ldr x19, [x8, x5] : phys_addr_t page_addr = page_to_phys(page); 0.00 : ffff8000104553c8: sub x20, x1, x7 : phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + 0.00 : ffff8000104553cc: sub x19, x19, x7 : phys_addr_t page_addr = page_to_phys(page); 0.00 : ffff8000104553d0: asr x20, x20, #6 : bv->bv_offset + bv->bv_len - 1; 0.00 : ffff8000104553d4: ldp w7, w5, [x24, #8] : phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + 0.00 : ffff8000104553d8: asr x19, x19, #6 : phys_addr_t page_addr = page_to_phys(page); 0.00 : ffff8000104553dc: lsl x20, x20, #12 : bv->bv_offset + bv->bv_len - 1; 0.00 : ffff8000104553e0: add x5, x5, x7 : if (vec_end_addr + 1 != page_addr + off) 0.00 : ffff8000104553e4: add x3, x20, w3, uxtw : bv->bv_offset + bv->bv_len - 1; 0.00 : ffff8000104553e8: add x19, x5, x19, lsl #12 : if (vec_end_addr + 1 != page_addr + off) 0.00 : ffff8000104553ec: cmp x19, x3 0.00 : ffff8000104553f0: b.eq ffff800010455404 <__bio_try_merge_page+0x8c> // b.none 0.00 : ffff8000104553f4: ldp x19, x20, [x29, #16] 0.00 : ffff8000104553f8: ldr x21, [x29, #32] 0.00 : ffff8000104553fc: ldp x24, x25, [x29, #56] 0.00 : ffff800010455400: b ffff800010455390 <__bio_try_merge_page+0x18> 0.00 : ffff800010455404: str x26, [x29, #72] 0.00 : ffff800010455408: mov x26, x0 : if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) 0.00 : ffff80001045540c: adrp x0, ffff800011af8000 0.00 : ffff800010455410: stp x22, x23, [x29, #40] 0.00 : ffff800010455414: mov x23, x4 0.00 : ffff800010455418: mov w22, w2 0.00 : ffff80001045541c: ldr w0, [x0, #1432] 0.00 : ffff800010455420: cbnz w0, ffff8000104554b8 <__bio_try_merge_page+0x140> : phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + 0.00 : ffff800010455424: sub x19, x19, #0x1 : *same_page = ((vec_end_addr & PAGE_MASK) == page_addr); 0.00 : ffff800010455428: and x0, x19, #0xfffffffffffff000 0.00 : ffff80001045542c: cmp x20, x0 0.00 : ffff800010455430: cset w1, eq // eq = none 0.00 : ffff800010455434: strb w1, [x23] : if (!*same_page && pfn_to_page(PFN_DOWN(vec_end_addr)) + 1 != page) 0.00 : ffff800010455438: b.eq ffff800010455468 <__bio_try_merge_page+0xf0> // b.none 0.00 : ffff80001045543c: lsr x19, x19, #12 0.00 : ffff800010455440: ldr x1, [x25, #1880] 0.00 : ffff800010455444: add x0, x19, #0x1 0.00 : ffff800010455448: add x0, x1, x0, lsl #6 0.00 : ffff80001045544c: cmp x21, x0 0.00 : ffff800010455450: b.eq ffff800010455468 <__bio_try_merge_page+0xf0> // b.none 0.00 : ffff800010455454: ldp x19, x20, [x29, #16] 0.00 : ffff800010455458: ldp x21, x22, [x29, #32] 0.00 : ffff80001045545c: ldp x23, x24, [x29, #48] 0.00 : ffff800010455460: ldp x25, x26, [x29, #64] 0.00 : ffff800010455464: b ffff800010455390 <__bio_try_merge_page+0x18> : __bio_try_merge_page(): : if (bio->bi_iter.bi_size > UINT_MAX - len) 0.00 : ffff800010455468: ldr w1, [x26, #40] 0.00 : ffff80001045546c: mvn w0, w22 0.00 : ffff800010455470: cmp w1, w0 0.00 : ffff800010455474: b.hi ffff800010455454 <__bio_try_merge_page+0xdc> // b.pmore : bv->bv_len += len; 0.00 : ffff800010455478: ldr w1, [x24, #8] : return true; 0.00 : ffff80001045547c: mov w0, #0x1 // #1 : bv->bv_len += len; 0.00 : ffff800010455480: add w1, w1, w22 0.00 : ffff800010455484: str w1, [x24, #8] : bio->bi_iter.bi_size += len; 0.00 : ffff800010455488: ldr w2, [x26, #40] 0.00 : ffff80001045548c: add w2, w2, w22 0.00 : ffff800010455490: str w2, [x26, #40] : return true; 0.00 : ffff800010455494: ldp x19, x20, [x29, #16] 0.00 : ffff800010455498: ldp x21, x22, [x29, #32] 0.00 : ffff80001045549c: ldp x23, x24, [x29, #48] 0.00 : ffff8000104554a0: ldp x25, x26, [x29, #64] 0.00 : ffff8000104554a4: b ffff800010455394 <__bio_try_merge_page+0x1c> : if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) 0.00 : ffff8000104554a8: brk #0x800 : return false; 0.00 : ffff8000104554ac: mov w0, #0x0 // #0 : } 0.00 : ffff8000104554b0: ldp x29, x30, [sp], #80 0.00 : ffff8000104554b4: ret : page_is_mergeable(): : if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) 0.00 : ffff8000104554b8: mov x0, x24 0.00 : ffff8000104554bc: bl ffff800010682f70 0.00 : ffff8000104554c0: tst w0, #0xff 0.00 : ffff8000104554c4: b.ne ffff800010455424 <__bio_try_merge_page+0xac> // b.any 0.00 : ffff8000104554c8: b ffff800010455454 <__bio_try_merge_page+0xdc> Percent | Source code & Disassembly of vmlinux for cycles (11 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101b4898 : : irq_work_tick(): : irq_work_run_list(this_cpu_ptr(&lazy_list)); : } : EXPORT_SYMBOL_GPL(irq_work_run); : : void irq_work_tick(void) : { 0.00 : ffff8000101b4898: stp x29, x30, [sp, #-32]! 0.00 : ffff8000101b489c: mov x29, sp 0.00 : ffff8000101b48a0: str x19, [sp, #16] : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff8000101b48a4: mov x1, sp : irq_work_tick(): : struct llist_head *raised = this_cpu_ptr(&raised_list); 0.00 : ffff8000101b48a8: adrp x19, ffff800011526000 0.00 : ffff8000101b48ac: add x0, x19, #0x78 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000101b48b0: mrs x2, tpidr_el1 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101b48b4: ldr x3, [x0, x2] : irq_work_tick(): 0.00 : ffff8000101b48b8: add x0, x0, x2 : : if (!llist_empty(raised) && !arch_irq_work_has_interrupt()) 0.00 : ffff8000101b48bc: cbz x3, ffff8000101b48cc : arch_irq_work_has_interrupt(): : : #include : : static inline bool arch_irq_work_has_interrupt(void) : { : return !!__smp_cross_call; 0.00 : ffff8000101b48c0: adrp x2, ffff800011af2000 : irq_work_tick(): 0.00 : ffff8000101b48c4: ldr x2, [x2, #2248] 0.00 : ffff8000101b48c8: cbz x2, ffff8000101b48ec : irq_work_run_list(raised); : irq_work_run_list(this_cpu_ptr(&lazy_list)); 72.76 : ffff8000101b48cc: add x0, x19, #0x78 : __my_cpu_offset(): 0.00 : ffff8000101b48d0: mrs x1, tpidr_el1 : irq_work_tick(): 0.00 : ffff8000101b48d4: add x0, x0, #0x8 0.00 : ffff8000101b48d8: add x0, x0, x1 0.00 : ffff8000101b48dc: bl ffff8000101b44c8 : } 27.24 : ffff8000101b48e0: ldr x19, [sp, #16] 0.00 : ffff8000101b48e4: ldp x29, x30, [sp], #32 0.00 : ffff8000101b48e8: ret : irq_work_run_list(raised); 0.00 : ffff8000101b48ec: bl ffff8000101b44c8 0.00 : ffff8000101b48f0: mov x1, sp 0.00 : ffff8000101b48f4: b ffff8000101b48cc Percent | Source code & Disassembly of vmlinux for cycles (12 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001009bd98 : : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001009bd98: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 9.44 : ffff80001009bd9c: ldr w0, [x0, #16] : armv8pmu_stop(): : armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E); : } : : static void armv8pmu_stop(struct arm_pmu *cpu_pmu) : { : WARN_ON_ONCE(preemptible()); 0.00 : ffff80001009bda0: cbz w0, ffff80001009bdb8 : armv8pmu_pmcr_read(): : return read_sysreg(pmcr_el0); 0.00 : ffff80001009bda4: mrs x0, pmcr_el0 : armv8pmu_pmcr_write(): : isb(); 0.00 : ffff80001009bda8: isb : write_sysreg(val, pmcr_el0); 90.56 : ffff80001009bdac: and x0, x0, #0x7e 0.00 : ffff80001009bdb0: msr pmcr_el0, x0 : armv8pmu_stop(): : /* Disable all counters */ : armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E); : } 0.00 : ffff80001009bdb4: ret : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001009bdb8: mrs x1, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001009bdbc: and w0, w1, #0x80 : armv8pmu_stop(): : WARN_ON_ONCE(preemptible()); 0.00 : ffff80001009bdc0: cbnz w0, ffff80001009bda4 0.00 : ffff80001009bdc4: brk #0x800 : armv8pmu_pmcr_read(): : return read_sysreg(pmcr_el0); 0.00 : ffff80001009bdc8: mrs x0, pmcr_el0 : armv8pmu_pmcr_write(): : isb(); 0.00 : ffff80001009bdcc: isb : write_sysreg(val, pmcr_el0); 0.00 : ffff80001009bdd0: and x0, x0, #0x7e 0.00 : ffff80001009bdd4: msr pmcr_el0, x0 : armv8pmu_stop(): : } 0.00 : ffff80001009bdd8: ret Percent | Source code & Disassembly of vmlinux for cycles (24 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010122dd0 : : update_nohz_stats(): : static bool update_nohz_stats(struct rq *rq, bool force) : { : #ifdef CONFIG_NO_HZ_COMMON : unsigned int cpu = rq->cpu; : : if (!rq->has_blocked_load) 0.00 : ffff800010122dd0: ldr w2, [x0, #40] 0.00 : ffff800010122dd4: cbz w2, ffff800010122e5c : unsigned int cpu = rq->cpu; 4.75 : ffff800010122dd8: ldr w3, [x0, #2568] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010122ddc: adrp x5, ffff800011afb000 11.24 : ffff800010122de0: add x5, x5, #0xfc0 : update_nohz_stats(): : return false; 0.00 : ffff800010122de4: mov w4, #0x0 // #0 : test_bit(): 34.13 : ffff800010122de8: add w2, w3, #0x3f 0.00 : ffff800010122dec: cmp w3, #0x0 0.00 : ffff800010122df0: csel w2, w2, w3, lt // lt = tstop 0.00 : ffff800010122df4: asr w2, w2, #6 0.00 : ffff800010122df8: sxtw x2, w2 0.00 : ffff800010122dfc: ldr x2, [x5, x2, lsl #3] 9.24 : ffff800010122e00: lsr x2, x2, x3 : update_nohz_stats(): : : if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 0.00 : ffff800010122e04: tbz w2, #0, ffff800010122e60 0.00 : ffff800010122e08: and w1, w1, #0xff : return false; : : if (!force && !time_after(jiffies, rq->last_blocked_load_update_tick)) 0.00 : ffff800010122e0c: cbnz w1, ffff800010122e28 0.00 : ffff800010122e10: adrp x2, ffff800011907000 0.00 : ffff800010122e14: ldr x1, [x0, #32] : return true; 0.00 : ffff800010122e18: mov w4, #0x1 // #1 : if (!force && !time_after(jiffies, rq->last_blocked_load_update_tick)) 0.00 : ffff800010122e1c: ldr x2, [x2, #2432] 0.00 : ffff800010122e20: sub x1, x1, x2 0.00 : ffff800010122e24: tbz x1, #63, ffff800010122e60 : { 0.00 : ffff800010122e28: stp x29, x30, [sp, #-32]! 0.00 : ffff800010122e2c: mov x29, sp 0.00 : ffff800010122e30: str x19, [sp, #16] 0.00 : ffff800010122e34: mov x19, x0 : : update_blocked_averages(cpu); 0.00 : ffff800010122e38: mov w0, w3 0.00 : ffff800010122e3c: bl ffff8000101226f8 : : return rq->has_blocked_load; 0.00 : ffff800010122e40: ldr w0, [x19, #40] : #else : return false; : #endif : } 0.00 : ffff800010122e44: ldr x19, [sp, #16] : return rq->has_blocked_load; 0.00 : ffff800010122e48: cmp w0, #0x0 0.00 : ffff800010122e4c: cset w4, ne // ne = any : } 0.00 : ffff800010122e50: mov w0, w4 0.00 : ffff800010122e54: ldp x29, x30, [sp], #32 0.00 : ffff800010122e58: ret : return false; 40.64 : ffff800010122e5c: mov w4, #0x0 // #0 : } 0.00 : ffff800010122e60: mov w0, w4 0.00 : ffff800010122e64: ret Percent | Source code & Disassembly of vmlinux for cycles (40 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010488a68 : : iov_iter_get_pages(): : } : : ssize_t iov_iter_get_pages(struct iov_iter *i, : struct page **pages, size_t maxsize, unsigned maxpages, : size_t *start) : { 0.00 : ffff800010488a68: stp x29, x30, [sp, #-112]! 0.00 : ffff800010488a6c: mov x29, sp 0.00 : ffff800010488a70: stp x19, x20, [sp, #16] 0.00 : ffff800010488a74: adrp x20, ffff800011909000 0.00 : ffff800010488a78: stp x21, x22, [sp, #32] 0.00 : ffff800010488a7c: add x5, x20, #0x908 0.00 : ffff800010488a80: stp x23, x24, [sp, #48] 0.00 : ffff800010488a84: mov x21, x1 0.00 : ffff800010488a88: mov x23, x0 0.00 : ffff800010488a8c: mov x22, x4 22.65 : ffff800010488a90: ldr x6, [x5] 5.11 : ffff800010488a94: str x6, [x29, #104] 0.00 : ffff800010488a98: mov x6, #0x0 // #0 14.84 : ffff800010488a9c: ldr x1, [x0, #16] : iov_iter_is_pipe(): : return iov_iter_type(i) == ITER_BVEC; : } : : static inline bool iov_iter_is_pipe(const struct iov_iter *i) : { : return iov_iter_type(i) == ITER_PIPE; 7.60 : ffff800010488aa0: ldr w5, [x0] 0.00 : ffff800010488aa4: cmp x1, x2 : iov_iter_type(): : return i->type & ~(READ | WRITE); 0.00 : ffff800010488aa8: and w6, w5, #0xfffffffe 0.00 : ffff800010488aac: csel x24, x1, x2, ls // ls = plast : iov_iter_get_pages(): : if (maxsize > i->count) : maxsize = i->count; : : if (unlikely(iov_iter_is_pipe(i))) 0.00 : ffff800010488ab0: cmp w6, #0x20 0.00 : ffff800010488ab4: b.eq ffff800010488bb8 // b.none : return pipe_get_pages(i, pages, maxsize, maxpages, start); : if (unlikely(iov_iter_is_discard(i))) 0.00 : ffff800010488ab8: cmp w6, #0x40 0.00 : ffff800010488abc: b.eq ffff800010488ba8 // b.none : return -EFAULT; : : iterate_all_kinds(i, maxsize, v, ({ 12.52 : ffff800010488ac0: cbz x24, ffff800010488bb0 0.00 : ffff800010488ac4: ldr x0, [x0, #8] 0.00 : ffff800010488ac8: tbnz w5, #4, ffff800010488cf8 10.01 : ffff800010488acc: tbnz w5, #3, ffff800010488d78 2.52 : ffff800010488ad0: tbnz w5, #6, ffff800010488bb0 2.50 : ffff800010488ad4: ldr x4, [x23, #24] 0.00 : ffff800010488ad8: ubfiz x6, x3, #12, #32 0.00 : ffff800010488adc: ldr x19, [x4, #8] 0.00 : ffff800010488ae0: sub x19, x19, x0 0.00 : ffff800010488ae4: cmp x19, x24 0.00 : ffff800010488ae8: csel x5, x19, x24, ls // ls = plast 7.55 : ffff800010488aec: cbz x5, ffff800010488b74 0.00 : ffff800010488af0: ldr x1, [x4] 0.00 : ffff800010488af4: mov x3, x21 0.00 : ffff800010488af8: add x0, x1, x0 0.00 : ffff800010488afc: and x19, x0, #0xfff 12.41 : ffff800010488b00: and x0, x0, #0xfffffffffffff000 0.00 : ffff800010488b04: str x19, [x22] 0.00 : ffff800010488b08: add x19, x19, x5 0.00 : ffff800010488b0c: cmp x19, x6 : iov_iter_rw(): : return iov_iter_type(i) == ITER_DISCARD; : } : : static inline unsigned char iov_iter_rw(const struct iov_iter *i) : { : return i->type & (READ | WRITE); 2.30 : ffff800010488b10: ldr w2, [x23] 0.00 : ffff800010488b14: csel x19, x19, x6, ls // ls = plast : iov_iter_get_pages(): 0.00 : ffff800010488b18: add x21, x19, #0xfff 0.00 : ffff800010488b1c: mvn w2, w2 0.00 : ffff800010488b20: and w2, w2, #0x1 0.00 : ffff800010488b24: lsr x21, x21, #12 0.00 : ffff800010488b28: mov w1, w21 0.00 : ffff800010488b2c: bl ffff800010207308 0.00 : ffff800010488b30: tbnz w0, #31, ffff800010488db4 0.00 : ffff800010488b34: sbfiz x1, x0, #12, #32 0.00 : ffff800010488b38: cmp w0, w21 0.00 : ffff800010488b3c: ldr x0, [x22] 0.00 : ffff800010488b40: csel x19, x1, x19, ne // ne = any 0.00 : ffff800010488b44: sub x19, x19, x0 : }),({ : return -EFAULT; : }) : ) : return 0; : } 0.00 : ffff800010488b48: add x1, x20, #0x908 0.00 : ffff800010488b4c: mov x0, x19 0.00 : ffff800010488b50: ldr x2, [x29, #104] 0.00 : ffff800010488b54: ldr x1, [x1] 0.00 : ffff800010488b58: eor x1, x2, x1 0.00 : ffff800010488b5c: cbnz x1, ffff800010488e2c 0.00 : ffff800010488b60: ldp x19, x20, [sp, #16] 0.00 : ffff800010488b64: ldp x21, x22, [sp, #32] 0.00 : ffff800010488b68: ldp x23, x24, [sp, #48] 0.00 : ffff800010488b6c: ldp x29, x30, [sp], #112 0.00 : ffff800010488b70: ret : iterate_all_kinds(i, maxsize, v, ({ 0.00 : ffff800010488b74: add x4, x4, #0x10 0.00 : ffff800010488b78: ldr x0, [x4, #8] 0.00 : ffff800010488b7c: cmp x0, x24 0.00 : ffff800010488b80: csel x0, x0, x24, ls // ls = plast 0.00 : ffff800010488b84: cbz x0, ffff800010488b74 0.00 : ffff800010488b88: ldr x1, [x4] 0.00 : ffff800010488b8c: mov x3, x21 0.00 : ffff800010488b90: and x19, x1, #0xfff 0.00 : ffff800010488b94: str x19, [x22] 0.00 : ffff800010488b98: add x19, x19, x0 0.00 : ffff800010488b9c: and x0, x1, #0xfffffffffffff000 0.00 : ffff800010488ba0: b ffff800010488b0c 0.00 : ffff800010488ba4: ldp x25, x26, [x29, #64] : pipe_get_pages(): : return -EFAULT; 0.00 : ffff800010488ba8: mov x19, #0xfffffffffffffff2 // #-14 0.00 : ffff800010488bac: b ffff800010488b48 : return 0; 0.00 : ffff800010488bb0: mov x19, #0x0 // #0 0.00 : ffff800010488bb4: b ffff800010488b48 0.00 : ffff800010488bb8: mov x19, #0x0 // #0 : if (!maxsize) 0.00 : ffff800010488bbc: cbz x24, ffff800010488b48 0.00 : ffff800010488bc0: str w3, [x29, #92] : if (!sanity(i)) 0.00 : ffff800010488bc4: bl ffff800010487f20 0.00 : ffff800010488bc8: ldr w3, [x29, #92] 0.00 : ffff800010488bcc: tst w0, #0xff 0.00 : ffff800010488bd0: b.eq ffff800010488ba8 // b.none 0.00 : ffff800010488bd4: stp x25, x26, [x29, #64] : data_start(): : size_t off = i->iov_offset; 0.00 : ffff800010488bd8: ldr x4, [x23, #8] : unsigned int iter_head = i->head; 0.00 : ffff800010488bdc: ldr w5, [x23, #32] : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || 0.00 : ffff800010488be0: cbz x4, ffff800010488c24 : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff800010488be4: ldr x2, [x23, #24] : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || 0.00 : ffff800010488be8: mov w7, #0x28 // #40 0.00 : ffff800010488bec: adrp x1, ffff800010d18000 0.00 : ffff800010488bf0: add x1, x1, #0xf80 0.00 : ffff800010488bf4: mov x6, #0x1000 // #4096 : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff800010488bf8: ldr w0, [x2, #92] : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || 0.00 : ffff800010488bfc: ldr x2, [x2, #144] : unsigned int p_mask = i->pipe->ring_size - 1; 0.00 : ffff800010488c00: sub w0, w0, #0x1 : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || 0.00 : ffff800010488c04: and w0, w0, w5 0.00 : ffff800010488c08: umaddl x0, w0, w7, x2 0.00 : ffff800010488c0c: ldr x0, [x0, #16] 0.00 : ffff800010488c10: cmp x0, x1 0.00 : ffff800010488c14: ccmp x4, x6, #0x4, eq // eq = none 0.00 : ffff800010488c18: b.ne ffff800010488c24 // b.any : iter_head++; 0.00 : ffff800010488c1c: add w5, w5, #0x1 : off = 0; 0.00 : ffff800010488c20: mov x4, #0x0 // #0 : *offp = off; 0.00 : ffff800010488c24: str x4, [x22] : pipe_get_pages(): : npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); 0.00 : ffff800010488c28: ldr x25, [x23, #24] 0.00 : ffff800010488c2c: ldp w0, w1, [x25, #84] 0.00 : ffff800010488c30: ldr w26, [x25, #92] : pipe_occupancy(): : * @head: The pipe ring head pointer : * @tail: The pipe ring tail pointer : */ : static inline unsigned int pipe_occupancy(unsigned int head, unsigned int tail) : { : return head - tail; 0.00 : ffff800010488c34: sub w0, w5, w0 : pipe_space_for_user(): : struct pipe_inode_info *pipe) : { : unsigned int p_occupancy, p_space; : : p_occupancy = pipe_occupancy(head, tail); : if (p_occupancy >= pipe->max_usage) 0.00 : ffff800010488c38: cmp w1, w0 0.00 : ffff800010488c3c: b.hi ffff800010488dbc // b.pmore 0.00 : ffff800010488c40: mov x1, #0x0 // #0 0.00 : ffff800010488c44: add x2, x29, #0x70 : pipe_get_pages(): : capacity = min(npages, maxpages) * PAGE_SIZE - *start; 0.00 : ffff800010488c48: sub x1, x1, x4 : return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start); 0.00 : ffff800010488c4c: cmp x1, x24 : __pipe_get_pages(): : ssize_t n = push_pipe(i, maxsize, &iter_head, start); 0.00 : ffff800010488c50: mov x3, x22 0.00 : ffff800010488c54: csel x1, x1, x24, ls // ls = plast 0.00 : ffff800010488c58: mov x0, x23 0.00 : ffff800010488c5c: str w5, [x2, #-12]! : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff800010488c60: sub w26, w26, #0x1 : ssize_t n = push_pipe(i, maxsize, &iter_head, start); 0.00 : ffff800010488c64: bl ffff800010488030 0.00 : ffff800010488c68: mov x19, x0 : if (!n) 0.00 : ffff800010488c6c: cbz x0, ffff800010488ba4 : n += *start; 0.00 : ffff800010488c70: ldr x0, [x22] 0.00 : ffff800010488c74: add x0, x19, x0 : while (n > 0) { 0.00 : ffff800010488c78: cmp x0, #0x0 0.00 : ffff800010488c7c: b.le ffff800010488cf0 0.00 : ffff800010488c80: sub x2, x0, #0x1 0.00 : ffff800010488c84: add x1, x21, #0x8 0.00 : ffff800010488c88: ldr w0, [x29, #100] : get_page(*pages++ = pipe->bufs[iter_head & p_mask].page); 0.00 : ffff800010488c8c: mov w4, #0x28 // #40 0.00 : ffff800010488c90: lsr x2, x2, #12 : __lse_atomic_add(): : } : : ATOMIC_OP(andnot, stclr) : ATOMIC_OP(or, stset) : ATOMIC_OP(xor, steor) : ATOMIC_OP(add, stadd) 0.00 : ffff800010488c94: mov w5, #0x1 // #1 0.00 : ffff800010488c98: add x2, x1, x2, lsl #3 0.00 : ffff800010488c9c: nop : __pipe_get_pages(): 0.00 : ffff800010488ca0: and w0, w0, w26 0.00 : ffff800010488ca4: ldr x1, [x25, #144] 0.00 : ffff800010488ca8: umull x0, w0, w4 0.00 : ffff800010488cac: ldr x0, [x1, x0] 0.00 : ffff800010488cb0: str x0, [x21] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010488cb4: ldr x1, [x0, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff800010488cb8: sub x3, x1, #0x1 0.00 : ffff800010488cbc: tst x1, #0x1 0.00 : ffff800010488cc0: csel x0, x3, x0, ne // ne = any : page_ref_inc(): : __page_ref_mod(page, -nr); : } : : static inline void page_ref_inc(struct page *page) : { : atomic_inc(&page->_refcount); 0.00 : ffff800010488cc4: add x0, x0, #0x34 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010488cc8: b ffff800010488dac 0.00 : ffff800010488ccc: b ffff800010488dac : __lse_atomic_add(): 0.00 : ffff800010488cd0: mov w1, w5 0.00 : ffff800010488cd4: stadd w1, [x0] : __pipe_get_pages(): : iter_head++; 0.00 : ffff800010488cd8: ldr w0, [x29, #100] 0.00 : ffff800010488cdc: add x21, x21, #0x8 : while (n > 0) { 0.00 : ffff800010488ce0: cmp x2, x21 : iter_head++; 0.00 : ffff800010488ce4: add w0, w0, #0x1 0.00 : ffff800010488ce8: str w0, [x29, #100] : while (n > 0) { 0.00 : ffff800010488cec: b.ne ffff800010488ca0 // b.any 0.00 : ffff800010488cf0: ldp x25, x26, [x29, #64] 0.00 : ffff800010488cf4: b ffff800010488b48 : iov_iter_get_pages(): : iterate_all_kinds(i, maxsize, v, ({ 0.00 : ffff800010488cf8: mov w8, w24 0.00 : ffff800010488cfc: cbz w24, ffff800010488bb0 0.00 : ffff800010488d00: ldr x4, [x23, #24] 0.00 : ffff800010488d04: mov w5, #0x0 // #0 0.00 : ffff800010488d08: mov w9, #0x1000 // #4096 0.00 : ffff800010488d0c: ldp w2, w1, [x4, #8] 0.00 : ffff800010488d10: ldr x10, [x4] 0.00 : ffff800010488d14: add w1, w0, w1 0.00 : ffff800010488d18: sub w7, w2, w0 0.00 : ffff800010488d1c: cmp w7, w8 0.00 : ffff800010488d20: and w3, w1, #0xfff 0.00 : ffff800010488d24: csel x19, x7, x8, ls // ls = plast 0.00 : ffff800010488d28: sub w6, w9, w3 0.00 : ffff800010488d2c: cmp w19, w6 0.00 : ffff800010488d30: csel x19, x19, x6, ls // ls = plast 0.00 : ffff800010488d34: cbz w19, ffff800010488dd8 0.00 : ffff800010488d38: lsr w1, w1, #12 0.00 : ffff800010488d3c: str x3, [x22] 0.00 : ffff800010488d40: add x1, x10, x1, lsl #6 0.00 : ffff800010488d44: str x1, [x21] : __read_once_size(): 0.00 : ffff800010488d48: ldr x0, [x1, #8] : compound_head(): 0.00 : ffff800010488d4c: sub x2, x0, #0x1 0.00 : ffff800010488d50: tst x0, #0x1 0.00 : ffff800010488d54: csel x1, x2, x1, ne // ne = any : page_ref_inc(): 0.00 : ffff800010488d58: add x1, x1, #0x34 : arch_static_branch_jump(): 0.00 : ffff800010488d5c: b ffff800010488d70 0.00 : ffff800010488d60: b ffff800010488d70 : __lse_atomic_add(): 0.00 : ffff800010488d64: mov w0, #0x1 // #1 0.00 : ffff800010488d68: stadd w0, [x1] 0.00 : ffff800010488d6c: b ffff800010488b48 : __ll_sc_atomic_add(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff800010488d70: b ffff80001048d7b0 0.00 : ffff800010488d74: b ffff800010488b48 : iov_iter_get_pages(): 0.00 : ffff800010488d78: ldr x2, [x23, #24] 0.00 : ffff800010488d7c: ldr x1, [x2, #8] 0.00 : ffff800010488d80: sub x0, x1, x0 0.00 : ffff800010488d84: cmp x0, x24 0.00 : ffff800010488d88: csel x0, x0, x24, ls // ls = plast 0.00 : ffff800010488d8c: cbnz x0, ffff800010488ba8 0.00 : ffff800010488d90: add x2, x2, #0x10 0.00 : ffff800010488d94: ldr x0, [x2, #8] 0.00 : ffff800010488d98: cmp x0, x24 0.00 : ffff800010488d9c: csel x0, x0, x24, ls // ls = plast 0.00 : ffff800010488da0: cbz x0, ffff800010488d90 : pipe_get_pages(): : return -EFAULT; 0.00 : ffff800010488da4: mov x19, #0xfffffffffffffff2 // #-14 0.00 : ffff800010488da8: b ffff800010488b48 : __ll_sc_atomic_add(): 0.00 : ffff800010488dac: b ffff80001048d7c8 0.00 : ffff800010488db0: b ffff800010488cd8 : iov_iter_get_pages(): : iterate_all_kinds(i, maxsize, v, ({ 0.00 : ffff800010488db4: sxtw x19, w0 0.00 : ffff800010488db8: b ffff800010488b48 0.00 : ffff800010488dbc: cmp w3, w1 : pipe_space_for_user(): : return 0; : p_space = pipe->ring_size - p_occupancy; 0.00 : ffff800010488dc0: sub w0, w26, w0 0.00 : ffff800010488dc4: csel w1, w3, w1, ls // ls = plast 0.00 : ffff800010488dc8: cmp w0, w1 0.00 : ffff800010488dcc: csel w0, w0, w1, ls // ls = plast 0.00 : ffff800010488dd0: lsl x1, x0, #12 0.00 : ffff800010488dd4: b ffff800010488c44 : bvec_iter_advance(): : } : : iter->bi_size -= bytes; : bytes += iter->bi_bvec_done; : : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010488dd8: cbz w0, ffff800010488d1c 0.00 : ffff800010488ddc: cmp w0, w2 0.00 : ffff800010488de0: b.cs ffff800010488df0 // b.hs, b.nlast 0.00 : ffff800010488de4: b ffff800010488d1c 0.00 : ffff800010488de8: cmp w0, w2 0.00 : ffff800010488dec: b.cc ffff800010488e18 // b.lo, b.ul, b.last : bytes -= bv[idx].bv_len; : idx++; 0.00 : ffff800010488df0: add w5, w5, #0x1 : bytes -= bv[idx].bv_len; 0.00 : ffff800010488df4: sub w0, w0, w2 0.00 : ffff800010488df8: ubfiz x3, x5, #4, #32 0.00 : ffff800010488dfc: add x3, x4, x3 0.00 : ffff800010488e00: ldr w2, [x3, #8] : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010488e04: cbnz w0, ffff800010488de8 0.00 : ffff800010488e08: mov w7, w2 0.00 : ffff800010488e0c: ldr w1, [x3, #12] 0.00 : ffff800010488e10: ldr x10, [x3] 0.00 : ffff800010488e14: b ffff800010488d1c 0.00 : ffff800010488e18: ldr w1, [x3, #12] 0.00 : ffff800010488e1c: sub w7, w2, w0 0.00 : ffff800010488e20: ldr x10, [x3] 0.00 : ffff800010488e24: add w1, w0, w1 0.00 : ffff800010488e28: b ffff800010488d1c 0.00 : ffff800010488e2c: stp x25, x26, [x29, #64] : iov_iter_get_pages(): : } 0.00 : ffff800010488e30: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (28 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102c58f8 : : blkdev_read_iter(): : ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) : { : struct file *file = iocb->ki_filp; : struct inode *bd_inode = bdev_file_inode(file); : loff_t size = i_size_read(bd_inode); : loff_t pos = iocb->ki_pos; 39.41 : ffff8000102c58f8: ldp x2, x3, [x0] : bdev_file_inode(): : return file->f_mapping->host; 21.05 : ffff8000102c58fc: ldr x2, [x2, #240] : blkdev_read_iter(): : loff_t size = i_size_read(bd_inode); 0.00 : ffff8000102c5900: ldr x2, [x2] 3.73 : ffff8000102c5904: ldr x2, [x2, #80] : : if (pos >= size) 0.00 : ffff8000102c5908: cmp x3, x2 0.00 : ffff8000102c590c: b.ge ffff8000102c5938 // b.tcont : { 28.57 : ffff8000102c5910: stp x29, x30, [sp, #-16]! : return 0; : : size -= pos; 0.00 : ffff8000102c5914: sub x2, x2, x3 : { 0.00 : ffff8000102c5918: mov x29, sp : iov_iter_truncate(): : * count doesn't have to fit in size_t - comparison extends both : * operands to u64 here and any value that would be truncated by : * conversion in assignement is by definition greater than all : * values of size_t, including old i->count. : */ : if (i->count > count) 7.25 : ffff8000102c591c: ldr x4, [x1, #16] 0.00 : ffff8000102c5920: cmp x2, x4 0.00 : ffff8000102c5924: b.cs ffff8000102c592c // b.hs, b.nlast : i->count = count; 0.00 : ffff8000102c5928: str x2, [x1, #16] : blkdev_read_iter(): : iov_iter_truncate(to, size); : return generic_file_read_iter(iocb, to); 0.00 : ffff8000102c592c: bl ffff8000101d4b78 : } 0.00 : ffff8000102c5930: ldp x29, x30, [sp], #16 0.00 : ffff8000102c5934: ret : return 0; 0.00 : ffff8000102c5938: mov x0, #0x0 // #0 : } 0.00 : ffff8000102c593c: ret Percent | Source code & Disassembly of vmlinux for cycles (13 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101279d0 : : balance_fair(): : } : : static int : balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) : { : if (rq->nr_running) 8.05 : ffff8000101279d0: ldr w1, [x0, #4] 0.00 : ffff8000101279d4: cbz w1, ffff8000101279e0 : return 1; 32.21 : ffff8000101279d8: mov w0, #0x1 // #1 : : return newidle_balance(rq, rf) != 0; : } 0.00 : ffff8000101279dc: ret : { 29.74 : ffff8000101279e0: stp x29, x30, [sp, #-16]! 0.00 : ffff8000101279e4: mov x1, x2 0.00 : ffff8000101279e8: mov x29, sp : return newidle_balance(rq, rf) != 0; 0.00 : ffff8000101279ec: bl ffff800010127620 7.28 : ffff8000101279f0: cmp w0, #0x0 0.00 : ffff8000101279f4: cset w0, ne // ne = any : } 22.71 : ffff8000101279f8: ldp x29, x30, [sp], #16 0.00 : ffff8000101279fc: ret Percent | Source code & Disassembly of vmlinux for cycles (10 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b6c0 : : tick_nohz_stop_idle(): : *last_update_time = ktime_to_us(now); : : } : : static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) : { 19.25 : ffff80001017b6c0: stp x29, x30, [sp, #-32]! : update_ts_time_stats(smp_processor_id(), ts, now, NULL); 0.00 : ffff80001017b6c4: adrp x2, ffff80001151d000 0.00 : ffff80001017b6c8: add x2, x2, #0x18 0.00 : ffff80001017b6cc: mov x3, #0x0 // #0 : { 0.00 : ffff80001017b6d0: mov x29, sp 0.00 : ffff80001017b6d4: str x19, [sp, #16] 0.00 : ffff80001017b6d8: mov x19, x0 : update_ts_time_stats(smp_processor_id(), ts, now, NULL); 0.00 : ffff80001017b6dc: mov x0, x2 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 20.21 : ffff80001017b6e0: mrs x2, tpidr_el1 : tick_nohz_stop_idle(): 29.77 : ffff80001017b6e4: ldr w0, [x0, x2] 0.00 : ffff80001017b6e8: mov x2, x1 0.00 : ffff80001017b6ec: mov x1, x19 0.00 : ffff80001017b6f0: bl ffff80001017b260 : ts->idle_active = 0; 0.00 : ffff80001017b6f4: ldrb w0, [x19, #76] 0.00 : ffff80001017b6f8: and w0, w0, #0xfffffffb 30.77 : ffff80001017b6fc: strb w0, [x19, #76] : : sched_clock_idle_wakeup_event(); : } 0.00 : ffff80001017b700: ldr x19, [sp, #16] 0.00 : ffff80001017b704: ldp x29, x30, [sp], #32 0.00 : ffff80001017b708: ret Percent | Source code & Disassembly of vmlinux for cycles (28 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010454eb8 <__bio_add_page>: : __bio_add_page(): : * that @bio has space for another bvec. : */ : void __bio_add_page(struct bio *bio, struct page *page, : unsigned int len, unsigned int off) : { : struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt]; 0.00 : ffff800010454eb8: ldrh w4, [x0, #96] : bio_flagged(): : atomic_set(&bio->__bi_cnt, count); : } : : static inline bool bio_flagged(struct bio *bio, unsigned int bit) : { : return (bio->bi_flags & (1U << bit)) != 0; 0.00 : ffff800010454ebc: ldrh w8, [x0, #20] : __bio_add_page(): 0.00 : ffff800010454ec0: ldr x7, [x0, #104] 0.00 : ffff800010454ec4: ubfiz x6, x4, #4, #16 0.00 : ffff800010454ec8: add x5, x7, x6 : : WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 0.00 : ffff800010454ecc: tbnz w8, #1, ffff800010454f48 <__bio_add_page+0x90> : bio_full(): : if (bio->bi_vcnt >= bio->bi_max_vecs) 21.41 : ffff800010454ed0: ldrh w8, [x0, #98] 0.00 : ffff800010454ed4: cmp w8, w4 0.00 : ffff800010454ed8: b.hi ffff800010454f0c <__bio_add_page+0x54> // b.pmore : __bio_add_page(): : WARN_ON_ONCE(bio_full(bio, len)); 0.00 : ffff800010454edc: brk #0x800 : : bv->bv_page = page; 3.59 : ffff800010454ee0: str x1, [x7, x6] : bv->bv_offset = off; 32.16 : ffff800010454ee4: stp w2, w3, [x5, #8] : bv->bv_len = len; : : bio->bi_iter.bi_size += len; : bio->bi_vcnt++; 0.00 : ffff800010454ee8: ldrh w3, [x0, #96] : bio->bi_iter.bi_size += len; 0.00 : ffff800010454eec: ldr w4, [x0, #40] : : if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page))) 0.00 : ffff800010454ef0: ldrh w5, [x0, #20] : bio->bi_vcnt++; 0.00 : ffff800010454ef4: add w3, w3, #0x1 : bio->bi_iter.bi_size += len; 0.00 : ffff800010454ef8: add w2, w4, w2 0.00 : ffff800010454efc: str w2, [x0, #40] : bio->bi_vcnt++; 17.86 : ffff800010454f00: strh w3, [x0, #96] : if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page))) 0.00 : ffff800010454f04: tbz w5, #5, ffff800010454f20 <__bio_add_page+0x68> : bio_set_flag(bio, BIO_WORKINGSET); : } 17.86 : ffff800010454f08: ret : bio_full(): : if (bio->bi_iter.bi_size > UINT_MAX - len) 0.00 : ffff800010454f0c: ldr w8, [x0, #40] 0.00 : ffff800010454f10: mvn w4, w2 0.00 : ffff800010454f14: cmp w8, w4 0.00 : ffff800010454f18: b.ls ffff800010454ee0 <__bio_add_page+0x28> // b.plast 0.00 : ffff800010454f1c: b ffff800010454edc <__bio_add_page+0x24> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010454f20: ldr x2, [x1, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff800010454f24: sub x3, x2, #0x1 0.00 : ffff800010454f28: tst x2, #0x1 0.00 : ffff800010454f2c: csel x1, x3, x1, ne // ne = any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 3.57 : ffff800010454f30: ldr x1, [x1] : __bio_add_page(): : if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page))) 0.00 : ffff800010454f34: tst w1, #0x40 3.55 : ffff800010454f38: b.eq ffff800010454f08 <__bio_add_page+0x50> // b.none : bio_set_flag(): : } : : static inline void bio_set_flag(struct bio *bio, unsigned int bit) : { : bio->bi_flags |= (1U << bit); 0.00 : ffff800010454f3c: orr w5, w5, #0x20 0.00 : ffff800010454f40: strh w5, [x0, #20] : __bio_add_page(): : } 0.00 : ffff800010454f44: ret : WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 0.00 : ffff800010454f48: brk #0x800 0.00 : ffff800010454f4c: ldrh w4, [x0, #96] 0.00 : ffff800010454f50: b ffff800010454ed0 <__bio_add_page+0x18> Percent | Source code & Disassembly of vmlinux for cycles (21 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001008db58 : : do_notify_resume(): : restore_saved_sigmask(); : } : : asmlinkage void do_notify_resume(struct pt_regs *regs, : unsigned long thread_flags) : { 0.00 : ffff80001008db58: stp x29, x30, [sp, #-192]! 0.00 : ffff80001008db5c: mov x29, sp 0.00 : ffff80001008db60: stp x19, x20, [sp, #16] 0.00 : ffff80001008db64: mov x19, x1 0.00 : ffff80001008db68: stp x21, x22, [sp, #32] 0.00 : ffff80001008db6c: adrp x22, ffff800011909000 0.00 : ffff80001008db70: stp x23, x24, [sp, #48] 0.00 : ffff80001008db74: add x2, x22, #0x908 0.00 : ffff80001008db78: stp x25, x26, [sp, #64] : addr_limit_user_check(): : #ifdef TIF_FSCHECK : if (!test_thread_flag(TIF_FSCHECK)) : return; : #endif : : if (CHECK_DATA_CORRUPTION(!segment_eq(get_fs(), USER_DS), 0.00 : ffff80001008db7c: adrp x24, ffff8000111b8000 : do_notify_resume(): 0.00 : ffff80001008db80: str x27, [sp, #80] 0.00 : ffff80001008db84: mov x21, x0 : addr_limit_user_check(): 0.00 : ffff80001008db88: add x24, x24, #0xf70 0.00 : ffff80001008db8c: mov x23, #0xffffffffffff // #281474976710655 : do_notify_resume(): 0.00 : ffff80001008db90: ldr x0, [x2] 0.00 : ffff80001008db94: str x0, [x29, #184] 0.00 : ffff80001008db98: mov x0, #0x0 // #0 : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff80001008db9c: mov x20, #0x20 // #32 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001008dba0: mrs x0, sp_el0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001008dba4: ldr x1, [x0] : addr_limit_user_check(): : if (!test_thread_flag(TIF_FSCHECK)) 0.00 : ffff80001008dba8: tbz w1, #5, ffff80001008dbcc : if (CHECK_DATA_CORRUPTION(!segment_eq(get_fs(), USER_DS), 0.00 : ffff80001008dbac: ldr x0, [x0, #8] 0.00 : ffff80001008dbb0: cmp x0, x23 0.00 : ffff80001008dbb4: b.ne ffff80001008decc // b.any : get_current(): 0.00 : ffff80001008dbb8: mrs x0, sp_el0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001008dbbc: b ffff80001008dc48 0.00 : ffff80001008dbc0: b ffff80001008dc48 : __lse_atomic64_andnot(): : " " #asm_op " %[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) 0.00 : ffff80001008dbc4: mov x1, x20 0.00 : ffff80001008dbc8: stclr x1, [x0] : do_notify_resume(): : : do { : /* Check valid user FS if needed */ : addr_limit_user_check(); : : if (thread_flags & _TIF_NEED_RESCHED) { 0.00 : ffff80001008dbcc: tbz w19, #1, ffff80001008dc50 : arch_static_branch_jump(): 0.00 : ffff80001008dbd0: b ffff80001008de28 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff80001008dbd4: nop 0.00 : ffff80001008dbd8: mov x0, #0x80 // #128 : local_daif_restore(): : * So we don't need additional synchronization here. : */ : gic_write_pmr(pmr); : } : : write_sysreg(flags, daif); 0.00 : ffff80001008dbdc: msr daif, x0 : do_notify_resume(): : /* Unmask Debug and SError for the next task */ : local_daif_restore(DAIF_PROCCTX_NOIRQ); : : schedule(); 95.24 : ffff80001008dbe0: bl ffff800010cd2a78 : local_daif_mask(): : asm volatile( 4.76 : ffff80001008dbe4: msr daifset, #0xf : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff80001008dbe8: b ffff80001008dc30 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff80001008dbec: nop : get_current(): 0.00 : ffff80001008dbf0: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001008dbf4: ldr x19, [x0] : do_notify_resume(): : fpsimd_restore_current_state(); : } : : local_daif_mask(); : thread_flags = READ_ONCE(current_thread_info()->flags); : } while (thread_flags & _TIF_WORK_MASK); 0.00 : ffff80001008dbf8: tst x19, #0x3f 0.00 : ffff80001008dbfc: b.ne ffff80001008dba0 // b.any : } 0.00 : ffff80001008dc00: add x22, x22, #0x908 0.00 : ffff80001008dc04: ldr x1, [x29, #184] 0.00 : ffff80001008dc08: ldr x0, [x22] 0.00 : ffff80001008dc0c: eor x0, x1, x0 0.00 : ffff80001008dc10: cbnz x0, ffff80001008df88 0.00 : ffff80001008dc14: ldp x19, x20, [sp, #16] 0.00 : ffff80001008dc18: ldp x21, x22, [sp, #32] 0.00 : ffff80001008dc1c: ldp x23, x24, [sp, #48] 0.00 : ffff80001008dc20: ldp x25, x26, [sp, #64] 0.00 : ffff80001008dc24: ldr x27, [sp, #80] 0.00 : ffff80001008dc28: ldp x29, x30, [sp], #192 0.00 : ffff80001008dc2c: ret : test_bit(): 0.00 : ffff80001008dc30: adrp x0, ffff800011af2000 0.00 : ffff80001008dc34: ldr x0, [x0, #1320] : local_daif_mask(): : if (system_uses_irq_prio_masking()) 0.00 : ffff80001008dc38: tbz x0, #42, ffff80001008dbf0 : gic_write_pmr(): : return read_sysreg_s(SYS_ICC_PMR_EL1); : } : : static inline void gic_write_pmr(u32 val) : { : write_sysreg_s(val, SYS_ICC_PMR_EL1); 0.00 : ffff80001008dc3c: mov x0, #0xf0 // #240 0.00 : ffff80001008dc40: msr s3_0_c4_c6_0, x0 0.00 : ffff80001008dc44: b ffff80001008dbf0 : __ll_sc_atomic64_andnot(): 0.00 : ffff80001008dc48: b ffff80001008dfd4 : do_notify_resume(): : if (thread_flags & _TIF_NEED_RESCHED) { 0.00 : ffff80001008dc4c: tbnz w19, #1, ffff80001008dbd0 : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff80001008dc50: b ffff80001008ddcc : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff80001008dc54: nop : local_daif_restore(): : write_sysreg(flags, daif); 0.00 : ffff80001008dc58: msr daif, xzr : do_notify_resume(): : if (thread_flags & _TIF_SIGPENDING) 0.00 : ffff80001008dc5c: tbnz w19, #0, ffff80001008dc9c : if (thread_flags & _TIF_NOTIFY_RESUME) { 0.00 : ffff80001008dc60: tbnz w19, #2, ffff80001008dc70 : if (thread_flags & _TIF_FOREIGN_FPSTATE) 0.00 : ffff80001008dc64: tbz w19, #3, ffff80001008dbe4 : fpsimd_restore_current_state(); 0.00 : ffff80001008dc68: bl ffff800010085e78 0.00 : ffff80001008dc6c: b ffff80001008dbe4 : get_current(): 0.00 : ffff80001008dc70: mrs x0, sp_el0 : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff80001008dc74: b ffff80001008dde8 0.00 : ffff80001008dc78: b ffff80001008dde8 : __lse_atomic64_andnot(): 0.00 : ffff80001008dc7c: mov x1, #0x4 // #4 0.00 : ffff80001008dc80: stclr x1, [x0] : tracehook_notify_resume(): : /* : * The caller just cleared TIF_NOTIFY_RESUME. This barrier : * pairs with task_work_add()->set_notify_resume() after : * hlist_add_head(task->task_works); : */ : smp_mb__after_atomic(); 0.00 : ffff80001008dc84: dmb ish : get_current(): 0.00 : ffff80001008dc88: mrs x0, sp_el0 : tracehook_notify_resume(): : if (unlikely(current->task_works)) 0.00 : ffff80001008dc8c: ldr x0, [x0, #1728] 0.00 : ffff80001008dc90: cbz x0, ffff80001008de00 : task_work_run(); 0.00 : ffff80001008dc94: bl ffff80001010a330 0.00 : ffff80001008dc98: b ffff80001008de00 : do_signal(): : if (syscall) { 0.00 : ffff80001008dc9c: ldr w0, [x21, #280] 0.00 : ffff80001008dca0: cmn w0, #0x1 0.00 : ffff80001008dca4: b.eq ffff80001008deb8 // b.none : restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4); 0.00 : ffff80001008dca8: ldp x27, x0, [x21, #256] : forget_syscall(): : return regs->syscallno != NO_SYSCALL; : } : : static inline void forget_syscall(struct pt_regs *regs) : { : regs->syscallno = NO_SYSCALL; 0.00 : ffff80001008dcac: mov w1, #0xffffffff // #-1 : do_signal(): : retval = regs->regs[0]; 0.00 : ffff80001008dcb0: ldr x26, [x21] : restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4); 0.00 : ffff80001008dcb4: mov x25, #0x4 // #4 : forget_syscall(): 0.00 : ffff80001008dcb8: str w1, [x21, #280] : do_signal(): 0.00 : ffff80001008dcbc: tst x0, #0x20 0.00 : ffff80001008dcc0: mov x0, #0x2 // #2 0.00 : ffff80001008dcc4: csel x25, x25, x0, eq // eq = none : switch (retval) { 0.00 : ffff80001008dcc8: cmn w26, #0x204 : restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4); 0.00 : ffff80001008dccc: sub x25, x27, x25 : switch (retval) { 0.00 : ffff80001008dcd0: b.eq ffff80001008de8c // b.none 0.00 : ffff80001008dcd4: b.lt ffff80001008dce4 // b.tstop 0.00 : ffff80001008dcd8: add w0, w26, #0x202 0.00 : ffff80001008dcdc: cmp w0, #0x2 0.00 : ffff80001008dce0: b.ls ffff80001008de8c // b.plast : if (get_signal(&ksig)) { 0.00 : ffff80001008dce4: add x0, x29, #0x60 0.00 : ffff80001008dce8: bl ffff8000100f9180 0.00 : ffff80001008dcec: tst w0, #0xff : if (regs->pc == restart_addr && 0.00 : ffff80001008dcf0: ldr x0, [x21, #256] : if (get_signal(&ksig)) { 0.00 : ffff80001008dcf4: b.eq ffff80001008de4c // b.none : if (regs->pc == restart_addr && 0.00 : ffff80001008dcf8: cmp x25, x0 0.00 : ffff80001008dcfc: b.eq ffff80001008df20 // b.none : get_current(): 0.00 : ffff80001008dd00: mrs x25, sp_el0 : test_bit(): 0.00 : ffff80001008dd04: ldr x2, [x25] : sigmask_to_save(): : restore_saved_sigmask(); : } : : static inline sigset_t *sigmask_to_save(void) : { : sigset_t *res = ¤t->blocked; 0.00 : ffff80001008dd08: add x27, x25, #0x678 : __read_once_size(): 0.00 : ffff80001008dd0c: ldr w0, [x25, #16] : sigmask_to_save(): : if (unlikely(test_restore_sigmask())) : res = ¤t->saved_sigmask; 0.00 : ffff80001008dd10: add x1, x25, #0x688 : handle_signal(): : int usig = ksig->sig; 0.00 : ffff80001008dd14: ldr w26, [x29, #176] : sigmask_to_save(): 0.00 : ffff80001008dd18: tst x2, #0x100000 : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff80001008dd1c: add w0, w0, #0x1 : sigmask_to_save(): 0.00 : ffff80001008dd20: csel x27, x1, x27, ne // ne = any : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001008dd24: str w0, [x25, #16] : __set_bit(): : *p |= mask; 0.00 : ffff80001008dd28: ldr x0, [x25, #2320] 0.00 : ffff80001008dd2c: orr x0, x0, #0x2 0.00 : ffff80001008dd30: str x0, [x25, #2320] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001008dd34: ldr x0, [x25, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff80001008dd38: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001008dd3c: str w0, [x25, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001008dd40: cbz x0, ffff80001008deb0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001008dd44: ldr x0, [x25, #16] : __preempt_count_dec_and_test(): 0.00 : ffff80001008dd48: cbz x0, ffff80001008deb0 : get_current(): 0.00 : ffff80001008dd4c: mrs x0, sp_el0 : rseq_handle_notify_resume(): : void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs); : : static inline void rseq_handle_notify_resume(struct ksignal *ksig, : struct pt_regs *regs) : { : if (current->rseq) 0.00 : ffff80001008dd50: ldr x0, [x0, #2304] 0.00 : ffff80001008dd54: cbz x0, ffff80001008dd64 : __rseq_handle_notify_resume(ksig, regs); 0.00 : ffff80001008dd58: mov x1, x21 0.00 : ffff80001008dd5c: add x0, x29, #0x60 0.00 : ffff80001008dd60: bl ffff8000101cf658 <__rseq_handle_notify_resume> : get_current(): 0.00 : ffff80001008dd64: mrs x0, sp_el0 : test_bit(): : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001008dd68: ldr x0, [x0] : handle_signal(): : if (is_compat_task()) { 0.00 : ffff80001008dd6c: tbz w0, #22, ffff80001008df04 : if (ksig->ka.sa.sa_flags & SA_SIGINFO) 0.00 : ffff80001008dd70: ldr x0, [x29, #104] : ret = compat_setup_rt_frame(usig, ksig, oldset, regs); 0.00 : ffff80001008dd74: mov x3, x21 0.00 : ffff80001008dd78: mov x2, x27 0.00 : ffff80001008dd7c: add x1, x29, #0x60 : if (ksig->ka.sa.sa_flags & SA_SIGINFO) 0.00 : ffff80001008dd80: tbnz w0, #2, ffff80001008dee4 : ret = compat_setup_frame(usig, ksig, oldset, regs); 0.00 : ffff80001008dd84: mov w0, w26 0.00 : ffff80001008dd88: bl ffff800010099758 0.00 : ffff80001008dd8c: mov w27, w0 : get_current(): 0.00 : ffff80001008dd90: mrs x1, sp_el0 : handle_signal(): : ret |= !valid_user_regs(®s->user_regs, current); 0.00 : ffff80001008dd94: mov x0, x21 0.00 : ffff80001008dd98: bl ffff80001008b4a0 0.00 : ffff80001008dd9c: cmp w0, #0x0 0.00 : ffff80001008dda0: cset w26, eq // eq = none 0.00 : ffff80001008dda4: orr w26, w26, w27 : if (!ret) 0.00 : ffff80001008dda8: cbnz w26, ffff80001008ddb4 : user_fastforward_single_step(tsk); 0.00 : ffff80001008ddac: mov x0, x25 0.00 : ffff80001008ddb0: bl ffff800010084cc8 : signal_setup_done(ret, ksig, 0); 0.00 : ffff80001008ddb4: mov w2, #0x0 // #0 0.00 : ffff80001008ddb8: add x1, x29, #0x60 0.00 : ffff80001008ddbc: mov w0, w26 0.00 : ffff80001008ddc0: bl ffff8000100f8de8 : do_notify_resume(): : if (thread_flags & _TIF_NOTIFY_RESUME) { 0.00 : ffff80001008ddc4: tbz w19, #2, ffff80001008dc64 0.00 : ffff80001008ddc8: b ffff80001008dc70 : test_bit(): 0.00 : ffff80001008ddcc: adrp x0, ffff800011af2000 0.00 : ffff80001008ddd0: ldr x0, [x0, #1320] : local_daif_restore(): : if (system_uses_irq_prio_masking()) { 0.00 : ffff80001008ddd4: tbz x0, #42, ffff80001008dc58 : gic_write_pmr(): 0.00 : ffff80001008ddd8: mov x0, #0xe0 // #224 0.00 : ffff80001008dddc: msr s3_0_c4_c6_0, x0 : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff80001008dde0: nop 0.00 : ffff80001008dde4: b ffff80001008dc58 : __ll_sc_atomic64_andnot(): 0.00 : ffff80001008dde8: mov x1, #0x4 // #4 0.00 : ffff80001008ddec: b ffff80001008dfec : tracehook_notify_resume(): : smp_mb__after_atomic(); 0.00 : ffff80001008ddf0: dmb ish : get_current(): 0.00 : ffff80001008ddf4: mrs x0, sp_el0 : tracehook_notify_resume(): : if (unlikely(current->task_works)) 0.00 : ffff80001008ddf8: ldr x0, [x0, #1728] 0.00 : ffff80001008ddfc: cbnz x0, ffff80001008dc94 : key_put(current->cached_requested_key); : current->cached_requested_key = NULL; : } : #endif : : mem_cgroup_handle_over_high(); 0.00 : ffff80001008de00: bl ffff80001026d630 : blkcg_maybe_throttle_current(); 0.00 : ffff80001008de04: bl ffff80001047cea8 : get_current(): 0.00 : ffff80001008de08: mrs x0, sp_el0 : rseq_handle_notify_resume(): : if (current->rseq) 0.00 : ffff80001008de0c: ldr x0, [x0, #2304] 0.00 : ffff80001008de10: cbz x0, ffff80001008dc64 : __rseq_handle_notify_resume(ksig, regs); 0.00 : ffff80001008de14: mov x1, x21 0.00 : ffff80001008de18: mov x0, #0x0 // #0 0.00 : ffff80001008de1c: bl ffff8000101cf658 <__rseq_handle_notify_resume> : do_notify_resume(): : if (thread_flags & _TIF_FOREIGN_FPSTATE) 0.00 : ffff80001008de20: tbz w19, #3, ffff80001008dbe4 0.00 : ffff80001008de24: b ffff80001008dc68 : test_bit(): 0.00 : ffff80001008de28: adrp x0, ffff800011af2000 0.00 : ffff80001008de2c: ldr x0, [x0, #1320] : local_daif_restore(): : } else if (system_uses_irq_prio_masking()) { 0.00 : ffff80001008de30: tbz x0, #42, ffff80001008dbd8 : gic_write_pmr(): 0.00 : ffff80001008de34: mov x0, #0x60 // #96 0.00 : ffff80001008de38: msr s3_0_c4_c6_0, x0 0.00 : ffff80001008de3c: mov x0, #0x0 // #0 0.00 : ffff80001008de40: b ffff80001008dbdc : local_daif_restore(): : pmr_sync(); 0.00 : ffff80001008de44: dsb sy 0.00 : ffff80001008de48: b ffff80001008dc58 : do_signal(): : if (syscall && regs->pc == restart_addr) { 0.00 : ffff80001008de4c: cmp x25, x0 0.00 : ffff80001008de50: b.eq ffff80001008df3c // b.none : get_current(): 0.00 : ffff80001008de54: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff80001008de58: ldr x1, [x0] : test_and_clear_bit(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (!(READ_ONCE(*p) & mask)) 0.00 : ffff80001008de5c: tbz w1, #20, ffff80001008dc60 : arch_static_branch_jump(): : asm_volatile_goto( 0.00 : ffff80001008de60: b ffff80001008def4 0.00 : ffff80001008de64: b ffff80001008def4 : __lse_atomic64_fetch_andnot(): : ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \ : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) 0.00 : ffff80001008de68: mov x1, #0x100000 // #1048576 0.00 : ffff80001008de6c: ldclral x1, x1, [x0] 0.00 : ffff80001008de70: mov x0, x1 : restore_saved_sigmask(): : if (test_and_clear_restore_sigmask()) 0.00 : ffff80001008de74: tbz w0, #20, ffff80001008dc60 : get_current(): 0.00 : ffff80001008de78: mrs x0, sp_el0 : restore_saved_sigmask(): : __set_current_blocked(¤t->saved_sigmask); 0.00 : ffff80001008de7c: add x0, x0, #0x688 0.00 : ffff80001008de80: bl ffff8000100f6f00 <__set_current_blocked> : do_notify_resume(): : if (thread_flags & _TIF_NOTIFY_RESUME) { 0.00 : ffff80001008de84: tbz w19, #2, ffff80001008dc64 0.00 : ffff80001008de88: b ffff80001008dc70 : do_signal(): : regs->regs[0] = regs->orig_x0; 0.00 : ffff80001008de8c: ldr x0, [x21, #272] 0.00 : ffff80001008de90: str x0, [x21] : if (get_signal(&ksig)) { 0.00 : ffff80001008de94: add x0, x29, #0x60 : regs->pc = restart_addr; 0.00 : ffff80001008de98: str x25, [x21, #256] : if (get_signal(&ksig)) { 0.00 : ffff80001008de9c: bl ffff8000100f9180 0.00 : ffff80001008dea0: tst w0, #0xff : if (regs->pc == restart_addr && 0.00 : ffff80001008dea4: ldr x0, [x21, #256] : if (get_signal(&ksig)) { 0.00 : ffff80001008dea8: b.eq ffff80001008de4c // b.none 0.00 : ffff80001008deac: b ffff80001008dcf8 : rseq_signal_deliver(): : static inline void rseq_signal_deliver(struct ksignal *ksig, : struct pt_regs *regs) : { : preempt_disable(); : __set_bit(RSEQ_EVENT_SIGNAL_BIT, ¤t->rseq_event_mask); : preempt_enable(); 0.00 : ffff80001008deb0: bl ffff800010cd2e50 0.00 : ffff80001008deb4: b ffff80001008dd4c : do_signal(): 0.00 : ffff80001008deb8: add x0, x29, #0x60 0.00 : ffff80001008debc: bl ffff8000100f9180 0.00 : ffff80001008dec0: tst w0, #0xff 0.00 : ffff80001008dec4: b.ne ffff80001008dd00 // b.any 0.00 : ffff80001008dec8: b ffff80001008de54 : addr_limit_user_check(): 0.00 : ffff80001008decc: mov x0, x24 0.00 : ffff80001008ded0: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff80001008ded4: brk #0x800 : "Invalid address limit on user-mode return")) : force_sig(SIGKILL); 0.00 : ffff80001008ded8: mov w0, #0x9 // #9 0.00 : ffff80001008dedc: bl ffff8000100f8d18 0.00 : ffff80001008dee0: b ffff80001008dbb8 : handle_signal(): : ret = compat_setup_rt_frame(usig, ksig, oldset, regs); 0.00 : ffff80001008dee4: mov w0, w26 0.00 : ffff80001008dee8: bl ffff800010099580 0.00 : ffff80001008deec: mov w27, w0 0.00 : ffff80001008def0: b ffff80001008dd90 : __ll_sc_atomic64_fetch_andnot(): 0.00 : ffff80001008def4: mov x2, #0x100000 // #1048576 0.00 : ffff80001008def8: b ffff80001008e004 0.00 : ffff80001008defc: mov x0, x1 0.00 : ffff80001008df00: b ffff80001008de74 : handle_signal(): : ret = setup_rt_frame(usig, ksig, oldset, regs); 0.00 : ffff80001008df04: mov x2, x27 0.00 : ffff80001008df08: mov x3, x21 0.00 : ffff80001008df0c: add x1, x29, #0x60 0.00 : ffff80001008df10: mov w0, w26 0.00 : ffff80001008df14: bl ffff80001008cff0 0.00 : ffff80001008df18: mov w27, w0 0.00 : ffff80001008df1c: b ffff80001008dd90 : do_signal(): : retval == -ERESTART_RESTARTBLOCK || 0.00 : ffff80001008df20: and w0, w26, #0xfffffffd : if (regs->pc == restart_addr && 0.00 : ffff80001008df24: cmn w0, #0x204 0.00 : ffff80001008df28: b.ne ffff80001008df68 // b.any : regs->regs[0] = -EINTR; 0.00 : ffff80001008df2c: mov x0, #0xfffffffffffffffc // #-4 0.00 : ffff80001008df30: str x0, [x21] : regs->pc = continue_addr; 0.00 : ffff80001008df34: str x27, [x21, #256] 0.00 : ffff80001008df38: b ffff80001008dd00 : if (retval == -ERESTART_RESTARTBLOCK) 0.00 : ffff80001008df3c: cmn w26, #0x204 0.00 : ffff80001008df40: b.eq ffff80001008df50 // b.none : get_current(): 0.00 : ffff80001008df44: mrs x0, sp_el0 : do_signal(): : user_rewind_single_step(current); 0.00 : ffff80001008df48: bl ffff800010084ca8 0.00 : ffff80001008df4c: b ffff80001008de54 : get_current(): 0.00 : ffff80001008df50: mrs x0, sp_el0 : test_bit(): 0.00 : ffff80001008df54: ldr x0, [x0] : setup_restart_syscall(): : if (is_compat_task()) 0.00 : ffff80001008df58: tbnz w0, #22, ffff80001008df7c : regs->regs[8] = __NR_restart_syscall; 0.00 : ffff80001008df5c: mov x0, #0x80 // #128 0.00 : ffff80001008df60: str x0, [x21, #64] 0.00 : ffff80001008df64: b ffff80001008df44 : do_signal(): : retval == -ERESTART_RESTARTBLOCK || 0.00 : ffff80001008df68: cmn w26, #0x200 0.00 : ffff80001008df6c: b.ne ffff80001008dd00 // b.any : !(ksig.ka.sa.sa_flags & SA_RESTART)))) { 0.00 : ffff80001008df70: ldr x0, [x29, #104] : (retval == -ERESTARTSYS && 0.00 : ffff80001008df74: tbz w0, #28, ffff80001008df2c 0.00 : ffff80001008df78: b ffff80001008dd00 : setup_restart_syscall(): : compat_setup_restart_syscall(regs); 0.00 : ffff80001008df7c: mov x0, x21 0.00 : ffff80001008df80: bl ffff8000100998b0 0.00 : ffff80001008df84: b ffff80001008df44 : do_notify_resume(): : } 0.00 : ffff80001008df88: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (9 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012acd0 : : task_tick_rt(): : * goes along full dynticks. Therefore no local assumption can be made : * and everything must be accessed through the @rq and @curr passed in : * parameters. : */ : static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued) : { 10.80 : ffff80001012acd0: stp x29, x30, [sp, #-32]! 0.00 : ffff80001012acd4: mov x29, sp 32.93 : ffff80001012acd8: stp x19, x20, [sp, #16] 0.00 : ffff80001012acdc: mov x20, x0 0.00 : ffff80001012ace0: mov x19, x1 : struct sched_rt_entity *rt_se = &p->rt; : : update_curr_rt(rq); 0.00 : ffff80001012ace4: bl ffff80001012aab8 : update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1); 0.00 : ffff80001012ace8: mov w2, #0x1 // #1 0.00 : ffff80001012acec: mov x1, x20 : rq_clock_pelt(): : static inline u64 rq_clock_pelt(struct rq *rq) : { : lockdep_assert_held(&rq->lock); : assert_clock_updated(rq); : : return rq->clock_pelt - rq->lost_idle_time; 0.00 : ffff80001012acf0: ldr x3, [x20, #2440] 0.00 : ffff80001012acf4: ldr x0, [x20, #2448] : task_tick_rt(): 0.00 : ffff80001012acf8: sub x0, x3, x0 0.00 : ffff80001012acfc: bl ffff800010134c40 : task_rlimit(): : } : : static inline unsigned long task_rlimit(const struct task_struct *task, : unsigned int limit) : { : return READ_ONCE(task->signal->rlim[limit].rlim_cur); 0.00 : ffff80001012ad00: ldr x1, [x19, #1640] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001012ad04: ldr x0, [x1, #912] 22.23 : ffff80001012ad08: ldr x2, [x1, #920] : watchdog(): : if (soft != RLIM_INFINITY) { 0.00 : ffff80001012ad0c: cmn x0, #0x1 0.00 : ffff80001012ad10: b.eq ffff80001012ad68 // b.none : if (p->rt.watchdog_stamp != jiffies) { 0.00 : ffff80001012ad14: adrp x3, ffff800011907000 0.00 : ffff80001012ad18: ldp x1, x5, [x19, #464] 0.00 : ffff80001012ad1c: ldr x4, [x3, #2432] 0.00 : ffff80001012ad20: cmp x5, x4 0.00 : ffff80001012ad24: b.eq ffff80001012ad34 // b.none : p->rt.watchdog_stamp = jiffies; 0.00 : ffff80001012ad28: ldr x3, [x3, #2432] : p->rt.timeout++; 0.00 : ffff80001012ad2c: add x1, x1, #0x1 : p->rt.watchdog_stamp = jiffies; 0.00 : ffff80001012ad30: stp x1, x3, [x19, #464] : next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ); 0.00 : ffff80001012ad34: cmp x0, x2 0.00 : ffff80001012ad38: mov x3, #0x3f7d // #16253 0.00 : ffff80001012ad3c: csel x0, x0, x2, ls // ls = plast 0.00 : ffff80001012ad40: movk x3, #0x5e35, lsl #16 0.00 : ffff80001012ad44: add x0, x0, #0xf9f 0.00 : ffff80001012ad48: movk x3, #0x49ba, lsl #32 0.00 : ffff80001012ad4c: movk x3, #0x20c, lsl #48 0.00 : ffff80001012ad50: lsr x0, x0, #5 0.00 : ffff80001012ad54: umulh x0, x0, x3 : if (p->rt.timeout > next) { 0.00 : ffff80001012ad58: cmp x0, x1 0.00 : ffff80001012ad5c: b.cs ffff80001012ad68 // b.hs, b.nlast : posix_cputimers_rt_watchdog(): : void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit); : : static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct, : u64 runtime) : { : pct->bases[CPUCLOCK_SCHED].nextevt = runtime; 0.00 : ffff80001012ad60: ldr x0, [x19, #272] 0.00 : ffff80001012ad64: str x0, [x19, #1504] : task_tick_rt(): : : /* : * RR tasks need a special form of timeslice management. : * FIFO tasks have no timeslices. : */ : if (p->policy != SCHED_RR) 11.52 : ffff80001012ad68: ldr w0, [x19, #728] 11.56 : ffff80001012ad6c: cmp w0, #0x2 0.00 : ffff80001012ad70: b.eq ffff80001012ad80 // b.none : requeue_task_rt(rq, p, 0); : resched_curr(rq); : return; : } : } : } 0.00 : ffff80001012ad74: ldp x19, x20, [sp, #16] 10.96 : ffff80001012ad78: ldp x29, x30, [sp], #32 0.00 : ffff80001012ad7c: ret : if (--p->rt.time_slice) 0.00 : ffff80001012ad80: ldr w0, [x19, #480] 0.00 : ffff80001012ad84: sub w0, w0, #0x1 0.00 : ffff80001012ad88: str w0, [x19, #480] 0.00 : ffff80001012ad8c: cbnz w0, ffff80001012ad74 : p->rt.time_slice = sched_rr_timeslice; 0.00 : ffff80001012ad90: adrp x0, ffff800011920000 : for_each_sched_rt_entity(rt_se) { 0.00 : ffff80001012ad94: adds x1, x19, #0x1c0 : p->rt.time_slice = sched_rr_timeslice; 0.00 : ffff80001012ad98: ldr w0, [x0, #2320] 0.00 : ffff80001012ad9c: str w0, [x19, #480] : for_each_sched_rt_entity(rt_se) { 0.00 : ffff80001012ada0: b.eq ffff80001012ad74 // b.none : if (rt_se->run_list.prev != rt_se->run_list.next) { 0.00 : ffff80001012ada4: ldr x2, [x1, #8] 0.00 : ffff80001012ada8: ldr x3, [x19, #448] 0.00 : ffff80001012adac: cmp x2, x3 0.00 : ffff80001012adb0: b.eq ffff80001012ad74 // b.none : requeue_rt_entity(): : if (on_rt_rq(rt_se)) { 0.00 : ffff80001012adb4: ldrh w4, [x19, #484] : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012adb8: adrp x0, ffff800011528000 : __read_once_size(): 0.00 : ffff80001012adbc: ldr w6, [x19, #68] : rq_of_rt_se(): 0.00 : ffff80001012adc0: add x0, x0, #0xe80 : requeue_rt_entity(): : if (on_rt_rq(rt_se)) { 0.00 : ffff80001012adc4: cbz w4, ffff80001012adfc : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012adc8: adrp x5, ffff800011909000 0.00 : ffff80001012adcc: add x5, x5, #0x928 : requeue_rt_entity(): : struct list_head *queue = array->queue + rt_se_prio(rt_se); 0.00 : ffff80001012add0: ldrsw x4, [x19, #108] : rq_of_rt_se(): : return task_rq(p); 0.00 : ffff80001012add4: ldr x5, [x5, w6, uxtw #3] : __list_del(): : * This is only for internal list manipulation where we know : * the prev/next entries already! : */ : static inline void __list_del(struct list_head * prev, struct list_head * next) : { : next->prev = prev; 0.00 : ffff80001012add8: str x2, [x3, #8] : rq_of_rt_se(): 0.00 : ffff80001012addc: add x0, x0, x5 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012ade0: str x3, [x2] : requeue_rt_entity(): : struct list_head *queue = array->queue + rt_se_prio(rt_se); 0.00 : ffff80001012ade4: add x0, x0, #0x210 0.00 : ffff80001012ade8: add x0, x0, x4, lsl #4 : list_add_tail(): : __list_add(new, head->prev, head); 0.00 : ffff80001012adec: ldr x2, [x0, #8] : __list_add(): : next->prev = new; 0.00 : ffff80001012adf0: str x1, [x0, #8] : new->prev = prev; 0.00 : ffff80001012adf4: stp x0, x2, [x19, #448] : __write_once_size(): 0.00 : ffff80001012adf8: str x1, [x2] : task_tick_rt(): : resched_curr(rq); 0.00 : ffff80001012adfc: mov x0, x20 0.00 : ffff80001012ae00: bl ffff800010114bb8 : return; 0.00 : ffff80001012ae04: b ffff80001012ad74 Percent | Source code & Disassembly of vmlinux for cycles (16 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010459f98 : : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010459f98: mrs x1, sp_el0 : blk_start_plug(): : struct task_struct *tsk = current; : : /* : * If this is a nested plug, don't actually assign it. : */ : if (tsk->plug) 0.00 : ffff800010459f9c: ldr x2, [x1, #1840] 0.00 : ffff800010459fa0: cbz x2, ffff800010459fa8 : /* : * Store ordering should not be needed here, since a potential : * preempt will imply a full memory barrier : */ : tsk->plug = plug; : } 0.00 : ffff800010459fa4: ret : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 62.89 : ffff800010459fa8: str x0, [x0] : blk_start_plug(): : INIT_LIST_HEAD(&plug->cb_list); 0.00 : ffff800010459fac: add x2, x0, #0x10 : INIT_LIST_HEAD(): : * the result is an empty list. : */ : static inline void INIT_LIST_HEAD(struct list_head *list) : { : WRITE_ONCE(list->next, list); : list->prev = list; 0.00 : ffff800010459fb0: str x0, [x0, #8] : __write_once_size(): 0.00 : ffff800010459fb4: str x2, [x0, #16] : INIT_LIST_HEAD(): 0.00 : ffff800010459fb8: str x2, [x0, #24] : blk_start_plug(): : plug->rq_count = 0; 5.45 : ffff800010459fbc: strh wzr, [x0, #32] : plug->multiple_queues = false; 19.04 : ffff800010459fc0: strb wzr, [x0, #34] : tsk->plug = plug; 6.31 : ffff800010459fc4: str x0, [x1, #1840] : } 6.32 : ffff800010459fc8: ret Percent | Source code & Disassembly of vmlinux for cycles (18 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc82b0 : : memchr_inv(): : * returns the address of the first character other than @c, or %NULL : * if the whole buffer contains just @c. : */ : void *memchr_inv(const void *start, int c, size_t bytes) : { : u8 value = c; 0.00 : ffff800010cc82b0: and w1, w1, #0xff : u64 value64; : unsigned int words, prefix; : : if (bytes <= 16) 0.00 : ffff800010cc82b4: cmp x2, #0x10 0.00 : ffff800010cc82b8: b.hi ffff800010cc8304 // b.pmore : return check_bytes8(start, value, bytes); 0.00 : ffff800010cc82bc: mov w3, w2 : check_bytes8(): : while (bytes) { 0.00 : ffff800010cc82c0: cbz x2, ffff800010cc82f8 : if (*start != value) 0.00 : ffff800010cc82c4: ldrb w2, [x0] 0.00 : ffff800010cc82c8: cmp w2, w1 0.00 : ffff800010cc82cc: b.ne ffff800010cc83d8 // b.any 39.35 : ffff800010cc82d0: sub w2, w3, #0x1 0.00 : ffff800010cc82d4: add x2, x2, #0x1 0.00 : ffff800010cc82d8: add x2, x0, x2 0.00 : ffff800010cc82dc: b ffff800010cc82ec 34.22 : ffff800010cc82e0: ldrb w3, [x0] 0.00 : ffff800010cc82e4: cmp w3, w1 0.00 : ffff800010cc82e8: b.ne ffff800010cc83d8 // b.any : start++; 26.43 : ffff800010cc82ec: add x0, x0, #0x1 : while (bytes) { 0.00 : ffff800010cc82f0: cmp x0, x2 0.00 : ffff800010cc82f4: b.ne ffff800010cc82e0 // b.any : return NULL; 0.00 : ffff800010cc82f8: mov x3, #0x0 // #0 : memchr_inv(): : start += 8; : words--; : } : : return check_bytes8(start, value, bytes % 8); : } 0.00 : ffff800010cc82fc: mov x0, x3 0.00 : ffff800010cc8300: ret : if (prefix) { 0.00 : ffff800010cc8304: ands w6, w0, #0x7 0.00 : ffff800010cc8308: b.ne ffff800010cc8394 // b.any : words = bytes / 8; 0.00 : ffff800010cc830c: lsr x3, x2, #3 : while (words) { 0.00 : ffff800010cc8310: cbz w3, ffff800010cc8354 : value64 = value; 0.00 : ffff800010cc8314: and x5, x1, #0xff : value64 *= 0x0101010101010101ULL; 0.00 : ffff800010cc8318: mov x6, #0x101010101010101 // #72340172838076673 : if (*(u64 *)start != value64) 0.00 : ffff800010cc831c: ldr x4, [x0] : value64 *= 0x0101010101010101ULL; 0.00 : ffff800010cc8320: mul x5, x5, x6 : if (*(u64 *)start != value64) 0.00 : ffff800010cc8324: cmp x5, x4 0.00 : ffff800010cc8328: b.ne ffff800010cc83e4 // b.any 0.00 : ffff800010cc832c: sub w3, w3, #0x1 0.00 : ffff800010cc8330: add x3, x3, #0x1 0.00 : ffff800010cc8334: add x3, x0, x3, lsl #3 0.00 : ffff800010cc8338: b ffff800010cc8348 0.00 : ffff800010cc833c: ldr x4, [x0] 0.00 : ffff800010cc8340: cmp x4, x5 0.00 : ffff800010cc8344: b.ne ffff800010cc83e4 // b.any : start += 8; 0.00 : ffff800010cc8348: add x0, x0, #0x8 : while (words) { 0.00 : ffff800010cc834c: cmp x0, x3 0.00 : ffff800010cc8350: b.ne ffff800010cc833c // b.any : check_bytes8(): : while (bytes) { 0.00 : ffff800010cc8354: ands w2, w2, #0x7 0.00 : ffff800010cc8358: b.eq ffff800010cc82f8 // b.none : if (*start != value) 0.00 : ffff800010cc835c: ldrb w3, [x0] 0.00 : ffff800010cc8360: cmp w3, w1 0.00 : ffff800010cc8364: b.ne ffff800010cc83d8 // b.any 0.00 : ffff800010cc8368: sub w2, w2, #0x1 0.00 : ffff800010cc836c: add x2, x2, #0x1 0.00 : ffff800010cc8370: add x2, x0, x2 0.00 : ffff800010cc8374: b ffff800010cc8384 0.00 : ffff800010cc8378: ldrb w3, [x0] 0.00 : ffff800010cc837c: cmp w3, w1 0.00 : ffff800010cc8380: b.ne ffff800010cc83d8 // b.any : start++; 0.00 : ffff800010cc8384: add x0, x0, #0x1 : while (bytes) { 0.00 : ffff800010cc8388: cmp x0, x2 0.00 : ffff800010cc838c: b.ne ffff800010cc8378 // b.any 0.00 : ffff800010cc8390: b ffff800010cc82f8 0.00 : ffff800010cc8394: mov w4, #0x7 // #7 0.00 : ffff800010cc8398: sub w4, w4, w6 : memchr_inv(): : prefix = 8 - prefix; 0.00 : ffff800010cc839c: mov w3, #0x8 // #8 0.00 : ffff800010cc83a0: add x4, x4, #0x1 0.00 : ffff800010cc83a4: sub w6, w3, w6 0.00 : ffff800010cc83a8: add x4, x0, x4 0.00 : ffff800010cc83ac: mov x3, x0 : check_bytes8(): : if (*start != value) 0.00 : ffff800010cc83b0: ldrb w5, [x3] 0.00 : ffff800010cc83b4: cmp w5, w1 0.00 : ffff800010cc83b8: b.ne ffff800010cc8404 // b.any : start++; 0.00 : ffff800010cc83bc: add x3, x3, #0x1 : while (bytes) { 0.00 : ffff800010cc83c0: cmp x3, x4 0.00 : ffff800010cc83c4: b.ne ffff800010cc83b0 // b.any : memchr_inv(): : start += prefix; 0.00 : ffff800010cc83c8: mov w3, w6 0.00 : ffff800010cc83cc: add x0, x0, x3 : bytes -= prefix; 0.00 : ffff800010cc83d0: sub x2, x2, x3 0.00 : ffff800010cc83d4: b ffff800010cc830c : check_bytes8(): : start++; 0.00 : ffff800010cc83d8: mov x3, x0 : memchr_inv(): : } 0.00 : ffff800010cc83dc: mov x0, x3 0.00 : ffff800010cc83e0: ret 0.00 : ffff800010cc83e4: add x3, x0, #0x8 : check_bytes8(): : if (*start != value) 0.00 : ffff800010cc83e8: ldrb w2, [x0] 0.00 : ffff800010cc83ec: cmp w2, w1 0.00 : ffff800010cc83f0: b.ne ffff800010cc83d8 // b.any : start++; 0.00 : ffff800010cc83f4: add x0, x0, #0x1 : while (bytes) { 0.00 : ffff800010cc83f8: cmp x0, x3 0.00 : ffff800010cc83fc: b.ne ffff800010cc83e8 // b.any 0.00 : ffff800010cc8400: b ffff800010cc82f8 : memchr_inv(): : if (r) 0.00 : ffff800010cc8404: cbz x3, ffff800010cc83c8 : } 0.00 : ffff800010cc8408: mov x0, x3 0.00 : ffff800010cc840c: ret Percent | Source code & Disassembly of vmlinux for cycles (9 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101346b8 <__accumulate_pelt_segments>: : __accumulate_pelt_segments(): : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); : return val; : } : : static u32 __accumulate_pelt_segments(u64 periods, u32 d1, u32 d3) : { 0.00 : ffff8000101346b8: mov w3, #0xb67e // #46718 : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.00 : ffff8000101346bc: cmp x0, #0x7e0 0.00 : ffff8000101346c0: add w2, w2, w3 0.00 : ffff8000101346c4: b.hi ffff8000101346fc <__accumulate_pelt_segments+0x44> // b.pmore : if (unlikely(local_n >= LOAD_AVG_PERIOD)) { 11.17 : ffff8000101346c8: cmp x0, #0x1f 0.00 : ffff8000101346cc: b.hi ffff800010134704 <__accumulate_pelt_segments+0x4c> // b.pmore : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 33.17 : ffff8000101346d0: adrp x3, ffff800010d06000 0.00 : ffff8000101346d4: add x3, x3, #0x450 : mul_u64_u32_shr(): : #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) : : #ifndef mul_u64_u32_shr : static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) : { : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff8000101346d8: mov w1, w1 : __accumulate_pelt_segments(): : u32 c1, c2, c3 = d3; /* y^0 == 1 */ : : /* : * c1 = d1 y^p : */ : c1 = decay_load((u64)d1, periods); 0.00 : ffff8000101346dc: mov x4, #0xba7e // #47742 : mul_u64_u32_shr(): 0.00 : ffff8000101346e0: ldr w5, [x3, w0, uxtw #2] 0.00 : ffff8000101346e4: mul x1, x1, x5 : __accumulate_pelt_segments(): 0.00 : ffff8000101346e8: lsr x1, x1, #32 : mul_u64_u32_shr(): 0.00 : ffff8000101346ec: mul x4, x4, x5 55.67 : ffff8000101346f0: add w2, w2, w1 0.00 : ffff8000101346f4: lsr x1, x4, #32 0.00 : ffff8000101346f8: sub w2, w2, w1 : __accumulate_pelt_segments(): : * n=0 n=p : */ : c2 = LOAD_AVG_MAX - decay_load(LOAD_AVG_MAX, periods) - 1024; : : return c1 + c2 + c3; : } 0.00 : ffff8000101346fc: mov w0, w2 0.00 : ffff800010134700: ret : decay_load(): : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010134704: and x6, x0, #0x1f 0.00 : ffff800010134708: adrp x5, ffff800010d06000 0.00 : ffff80001013470c: add x5, x5, #0x450 : val >>= local_n / LOAD_AVG_PERIOD; 0.00 : ffff800010134710: lsr w0, w0, #5 : __accumulate_pelt_segments(): : c1 = decay_load((u64)d1, periods); 0.00 : ffff800010134714: mov w3, w1 : decay_load(): : val >>= local_n / LOAD_AVG_PERIOD; 0.00 : ffff800010134718: mov x4, #0xba7e // #47742 0.00 : ffff80001013471c: lsr x4, x4, x0 : mul_u64_u32_shr(): 0.00 : ffff800010134720: ldr w5, [x5, x6, lsl #2] : decay_load(): 0.00 : ffff800010134724: lsr x3, x3, x0 : mul_u64_u32_shr(): 0.00 : ffff800010134728: mul x3, x3, x5 : __accumulate_pelt_segments(): : c1 = decay_load((u64)d1, periods); 0.00 : ffff80001013472c: lsr x1, x3, #32 0.00 : ffff800010134730: b ffff8000101346ec <__accumulate_pelt_segments+0x34> Percent | Source code & Disassembly of vmlinux for cycles (9 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010a82dc8 : : cpuidle_not_available(): : } : : bool cpuidle_not_available(struct cpuidle_driver *drv, : struct cpuidle_device *dev) : { : return off || !initialized || !drv || !dev || !dev->enabled; 0.00 : ffff800010a82dc8: adrp x2, ffff80001190e000 0.00 : ffff800010a82dcc: add x3, x2, #0xd84 100.00 : ffff800010a82dd0: ldr w3, [x3, #4] 0.00 : ffff800010a82dd4: cbnz w3, ffff800010a82e04 0.00 : ffff800010a82dd8: ldr w3, [x2, #3460] 0.00 : ffff800010a82ddc: mov w2, #0x1 // #1 0.00 : ffff800010a82de0: cbz w3, ffff800010a82dfc 0.00 : ffff800010a82de4: cmp x0, #0x0 0.00 : ffff800010a82de8: ccmp x1, #0x0, #0x4, ne // ne = any 0.00 : ffff800010a82dec: b.eq ffff800010a82dfc // b.none 0.00 : ffff800010a82df0: ldrb w2, [x1] 0.00 : ffff800010a82df4: eor x2, x2, #0x2 0.00 : ffff800010a82df8: ubfx w2, w2, #1, #1 : } 0.00 : ffff800010a82dfc: mov w0, w2 0.00 : ffff800010a82e00: ret 0.00 : ffff800010a82e04: mov w2, #0x1 // #1 0.00 : ffff800010a82e08: mov w0, w2 0.00 : ffff800010a82e0c: ret Percent | Source code & Disassembly of vmlinux for cycles (8 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010166f30 : : profile_tick(): : do_profile_hits(type, __pc, nr_hits); : } : EXPORT_SYMBOL_GPL(profile_hits); : : void profile_tick(int type) : { 0.00 : ffff800010166f30: stp x29, x30, [sp, #-32]! : get_irq_regs(): : */ : DECLARE_PER_CPU(struct pt_regs *, __irq_regs); : : static inline struct pt_regs *get_irq_regs(void) : { : return __this_cpu_read(__irq_regs); 0.00 : ffff800010166f34: adrp x1, ffff800011528000 0.00 : ffff800010166f38: add x1, x1, #0x780 : profile_tick(): 0.00 : ffff800010166f3c: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010166f40: mrs x2, tpidr_el1 : get_irq_regs(): 0.00 : ffff800010166f44: ldr x3, [x1, x2] : profile_tick(): : struct pt_regs *regs = get_irq_regs(); : : if (!user_mode(regs) && cpumask_available(prof_cpu_mask) && 0.00 : ffff800010166f48: ldr x1, [x3, #264] 0.00 : ffff800010166f4c: tst x1, #0xf 0.00 : ffff800010166f50: b.eq ffff800010166f94 // b.none 38.47 : ffff800010166f54: str x19, [x29, #16] : cpumask_test_cpu(smp_processor_id(), prof_cpu_mask)) 0.00 : ffff800010166f58: adrp x1, ffff80001151d000 0.00 : ffff800010166f5c: add x1, x1, #0x18 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010166f60: adrp x19, ffff800011b20000 <__log_buf+0x1fc10> : profile_tick(): 24.10 : ffff800010166f64: ldr w2, [x1, x2] : test_bit(): 0.00 : ffff800010166f68: add x19, x19, #0xe50 0.00 : ffff800010166f6c: add x4, x19, #0x18 0.00 : ffff800010166f70: add w1, w2, #0x3f 0.00 : ffff800010166f74: cmp w2, #0x0 0.00 : ffff800010166f78: csel w1, w1, w2, lt // lt = tstop 0.00 : ffff800010166f7c: asr w1, w1, #6 0.00 : ffff800010166f80: sxtw x1, w1 0.00 : ffff800010166f84: ldr x1, [x4, x1, lsl #3] 0.00 : ffff800010166f88: lsr x2, x1, x2 : profile_tick(): : if (!user_mode(regs) && cpumask_available(prof_cpu_mask) && 0.00 : ffff800010166f8c: tbnz w2, #0, ffff800010166f9c 37.42 : ffff800010166f90: ldr x19, [x29, #16] : profile_hit(type, (void *)profile_pc(regs)); : } 0.00 : ffff800010166f94: ldp x29, x30, [sp], #32 0.00 : ffff800010166f98: ret 0.00 : ffff800010166f9c: str x20, [x29, #24] 0.00 : ffff800010166fa0: mov w20, w0 : profile_hit(type, (void *)profile_pc(regs)); 0.00 : ffff800010166fa4: mov x0, x3 0.00 : ffff800010166fa8: bl ffff80001008e278 : profile_hit(): : static inline void profile_hit(int type, void *ip) : { : /* : * Speedup for the common (no profiling enabled) case: : */ : if (unlikely(prof_on == type)) 0.00 : ffff800010166fac: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010166fb0: ldr w1, [x1, #696] 0.00 : ffff800010166fb4: cmp w20, w1 0.00 : ffff800010166fb8: b.eq ffff800010166fc8 // b.none 0.00 : ffff800010166fbc: ldp x19, x20, [x29, #16] : profile_tick(): : } 0.00 : ffff800010166fc0: ldp x29, x30, [sp], #32 0.00 : ffff800010166fc4: ret : profile_hits(): : if (prof_on != type || !prof_buffer) 0.00 : ffff800010166fc8: ldr x1, [x19, #64] 0.00 : ffff800010166fcc: cbz x1, ffff800010166fbc : do_profile_hits(type, __pc, nr_hits); 0.00 : ffff800010166fd0: mov w1, #0x1 // #1 0.00 : ffff800010166fd4: bl ffff800010166cb8 0.00 : ffff800010166fd8: ldp x19, x20, [x29, #16] : profile_tick(): : } 0.00 : ffff800010166fdc: b ffff800010166f94 Percent | Source code & Disassembly of vmlinux for cycles (8 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101c4380 : : perf_event_task_tick(): : : return true; : } : : void perf_event_task_tick(void) : { 0.00 : ffff8000101c4380: stp x29, x30, [sp, #-96]! 0.00 : ffff8000101c4384: mov x29, sp 0.00 : ffff8000101c4388: stp x20, x21, [sp, #24] : __my_cpu_offset(): : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff8000101c438c: mov x4, sp : perf_event_task_tick(): 12.78 : ffff8000101c4390: stp x22, x23, [sp, #40] : struct list_head *head = this_cpu_ptr(&active_ctx_list); 0.00 : ffff8000101c4394: adrp x22, ffff800011526000 0.00 : ffff8000101c4398: add x22, x22, #0x98 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000101c439c: mrs x2, tpidr_el1 : perf_event_task_tick(): : struct perf_event_context *ctx, *tmp; : int throttled; : : lockdep_assert_irqs_disabled(); : : __this_cpu_inc(perf_throttled_seq); 0.00 : ffff8000101c43a0: add x1, x22, #0x40 : throttled = __this_cpu_xchg(perf_throttled_count, 0); 0.00 : ffff8000101c43a4: add x0, x22, #0x48 : __this_cpu_inc(perf_throttled_seq); 0.00 : ffff8000101c43a8: ldr x3, [x1, x2] : struct list_head *head = this_cpu_ptr(&active_ctx_list); 0.00 : ffff8000101c43ac: add x22, x22, #0xa0 : __this_cpu_inc(perf_throttled_seq); 0.00 : ffff8000101c43b0: add x3, x3, #0x1 0.00 : ffff8000101c43b4: str x3, [x1, x2] : __my_cpu_offset(): 0.00 : ffff8000101c43b8: mrs x1, tpidr_el1 : perf_event_task_tick(): : throttled = __this_cpu_xchg(perf_throttled_count, 0); 0.00 : ffff8000101c43bc: ldr w23, [x0, x1] 0.00 : ffff8000101c43c0: str wzr, [x0, x1] : tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); : : list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 12.64 : ffff8000101c43c4: ldr x21, [x22, x2] : struct list_head *head = this_cpu_ptr(&active_ctx_list); 0.00 : ffff8000101c43c8: add x22, x22, x2 : list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 0.00 : ffff8000101c43cc: cmp x22, x21 0.00 : ffff8000101c43d0: ldr x20, [x21], #-48 0.00 : ffff8000101c43d4: b.eq ffff8000101c45c4 // b.none 12.22 : ffff8000101c43d8: stp x24, x25, [x29, #56] : perf_adjust_period(): : period = perf_calculate_period(event, nsec, count); 0.00 : ffff8000101c43dc: mov x25, #0x900 // #2304 : event_filter_match(): : return (event->cpu == -1 || event->cpu == smp_processor_id()) && 0.00 : ffff8000101c43e0: adrp x24, ffff80001151d000 0.00 : ffff8000101c43e4: str x19, [x29, #16] 12.08 : ffff8000101c43e8: stp x26, x27, [x29, #72] 0.00 : ffff8000101c43ec: sub x20, x20, #0x30 0.00 : ffff8000101c43f0: add x24, x24, #0x18 : perf_adjust_period(): : period = perf_calculate_period(event, nsec, count); 0.00 : ffff8000101c43f4: movk x25, #0x3d, lsl #16 0.00 : ffff8000101c43f8: b ffff8000101c4414 : perf_event_task_tick(): : list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 0.00 : ffff8000101c43fc: mov x0, x20 0.00 : ffff8000101c4400: mov x21, x20 0.00 : ffff8000101c4404: ldr x20, [x0, #48]! 0.00 : ffff8000101c4408: sub x20, x20, #0x30 0.00 : ffff8000101c440c: cmp x22, x0 0.00 : ffff8000101c4410: b.eq ffff8000101c45b8 // b.none : perf_adjust_freq_unthr_context(): : if (!(ctx->nr_freq || needs_unthr)) 0.00 : ffff8000101c4414: ldr w0, [x21, #160] 0.00 : ffff8000101c4418: orr w0, w23, w0 0.00 : ffff8000101c441c: cbz w0, ffff8000101c43fc : raw_spin_lock(&ctx->lock); 12.23 : ffff8000101c4420: add x27, x21, #0x8 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101c4424: mov x26, x21 : perf_adjust_freq_unthr_context(): 0.00 : ffff8000101c4428: mov x0, x27 0.00 : ffff8000101c442c: bl ffff800010cd80f0 <_raw_spin_lock> : perf_pmu_disable(ctx->pmu); 38.05 : ffff8000101c4430: ldr x0, [x21] 0.00 : ffff8000101c4434: bl ffff8000101c2450 : __read_once_size(): 0.00 : ffff8000101c4438: ldr x19, [x26, #96]! : perf_adjust_freq_unthr_context(): : list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 0.00 : ffff8000101c443c: cmp x19, x26 0.00 : ffff8000101c4440: b.ne ffff8000101c4454 // b.any 0.00 : ffff8000101c4444: b ffff8000101c4590 : __read_once_size(): 0.00 : ffff8000101c4448: ldr x19, [x19] : perf_adjust_freq_unthr_context(): 0.00 : ffff8000101c444c: cmp x19, x26 0.00 : ffff8000101c4450: b.eq ffff8000101c4590 // b.none : if (event->state != PERF_EVENT_STATE_ACTIVE) 0.00 : ffff8000101c4454: ldr w0, [x19, #168] 0.00 : ffff8000101c4458: cmp w0, #0x1 0.00 : ffff8000101c445c: b.ne ffff8000101c4448 // b.any : event_filter_match(): : return (event->cpu == -1 || event->cpu == smp_processor_id()) && 0.00 : ffff8000101c4460: ldr w0, [x19, #636] 0.00 : ffff8000101c4464: mov x2, sp : perf_cgroup_match(event) && pmu_filter_match(event); 0.00 : ffff8000101c4468: cmn w0, #0x1 0.00 : ffff8000101c446c: b.eq ffff8000101c4484 // b.none : return (event->cpu == -1 || event->cpu == smp_processor_id()) && 0.00 : ffff8000101c4470: mov x1, x24 : __my_cpu_offset(): 0.00 : ffff8000101c4474: mrs x3, tpidr_el1 : event_filter_match(): 0.00 : ffff8000101c4478: ldr w1, [x1, x3] 0.00 : ffff8000101c447c: cmp w0, w1 0.00 : ffff8000101c4480: b.ne ffff8000101c4448 // b.any : perf_cgroup_match(): : struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 0.00 : ffff8000101c4484: ldr x3, [x19, #544] : if (!event->cgrp) 0.00 : ffff8000101c4488: ldr x1, [x19, #936] : __my_cpu_offset(): 0.00 : ffff8000101c448c: mrs x4, tpidr_el1 : __get_cpu_context(): : return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 0.00 : ffff8000101c4490: ldr x2, [x3] 0.00 : ffff8000101c4494: ldr x0, [x2, #72] : perf_cgroup_match(): : if (!event->cgrp) 0.00 : ffff8000101c4498: cbz x1, ffff8000101c44e8 : if (!cpuctx->cgrp) 0.00 : ffff8000101c449c: add x0, x0, x4 0.00 : ffff8000101c44a0: ldr x0, [x0, #360] 0.00 : ffff8000101c44a4: cbz x0, ffff8000101c4448 : return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 0.00 : ffff8000101c44a8: ldr x2, [x0] 0.00 : ffff8000101c44ac: ldr x0, [x1] : cgroup_is_descendant(): : * and @ancestor are accessible. : */ : static inline bool cgroup_is_descendant(struct cgroup *cgrp, : struct cgroup *ancestor) : { : if (cgrp->root != ancestor->root || cgrp->level < ancestor->level) 0.00 : ffff8000101c44b0: ldr x3, [x2, #488] 0.00 : ffff8000101c44b4: ldr x1, [x0, #488] 0.00 : ffff8000101c44b8: cmp x3, x1 0.00 : ffff8000101c44bc: b.ne ffff8000101c4448 // b.any 0.00 : ffff8000101c44c0: ldr w1, [x0, #248] 0.00 : ffff8000101c44c4: ldr w3, [x2, #248] 0.00 : ffff8000101c44c8: cmp w3, w1 0.00 : ffff8000101c44cc: b.lt ffff8000101c4448 // b.tstop : return false; : return cgrp->ancestor_ids[ancestor->level] == cgroup_id(ancestor); 0.00 : ffff8000101c44d0: add x1, x2, w1, sxtw #3 : cgroup_id(): : return cgrp->kn->id; 0.00 : ffff8000101c44d4: ldr x0, [x0, #288] : event_filter_match(): : return (event->cpu == -1 || event->cpu == smp_processor_id()) && 0.00 : ffff8000101c44d8: ldr x1, [x1, #896] 0.00 : ffff8000101c44dc: ldr x0, [x0, #104] 0.00 : ffff8000101c44e0: cmp x1, x0 0.00 : ffff8000101c44e4: b.ne ffff8000101c4448 // b.any : __pmu_filter_match(): : return pmu->filter_match ? pmu->filter_match(event) : 1; 0.00 : ffff8000101c44e8: ldr x0, [x19, #152] 0.00 : ffff8000101c44ec: ldr x1, [x0, #280] 0.00 : ffff8000101c44f0: cbz x1, ffff8000101c4500 0.00 : ffff8000101c44f4: mov x0, x19 0.00 : ffff8000101c44f8: blr x1 : pmu_filter_match(): : if (!__pmu_filter_match(event)) 0.00 : ffff8000101c44fc: cbz w0, ffff8000101c4448 0.00 : ffff8000101c4500: str x28, [x29, #88] : for_each_sibling_event(sibling, event) { 0.00 : ffff8000101c4504: ldr x0, [x19, #144] 0.00 : ffff8000101c4508: cmp x19, x0 0.00 : ffff8000101c450c: b.eq ffff8000101c4654 // b.none : perf_adjust_freq_unthr_context(): : perf_pmu_disable(event->pmu); 0.00 : ffff8000101c4510: ldr x0, [x19, #152] : if (hwc->interrupts == MAX_INTERRUPTS) { 0.00 : ffff8000101c4514: add x28, x19, #0x160 : perf_pmu_disable(event->pmu); 0.00 : ffff8000101c4518: bl ffff8000101c2450 : if (hwc->interrupts == MAX_INTERRUPTS) { 0.00 : ffff8000101c451c: ldr x0, [x28, #168] 0.00 : ffff8000101c4520: cmn x0, #0x1 0.00 : ffff8000101c4524: b.eq ffff8000101c45d4 // b.none : if (!event->attr.freq || !event->attr.sample_freq) 0.00 : ffff8000101c4528: ldrb w1, [x19, #265] 0.00 : ffff8000101c452c: ldr x0, [x19, #152] 0.00 : ffff8000101c4530: tbz w1, #2, ffff8000101c457c 0.00 : ffff8000101c4534: ldr x1, [x19, #240] 0.00 : ffff8000101c4538: cbz x1, ffff8000101c457c : event->pmu->stop(event, PERF_EF_UPDATE); 0.00 : ffff8000101c453c: ldr x2, [x0, #160] 0.00 : ffff8000101c4540: mov w1, #0x4 // #4 0.00 : ffff8000101c4544: mov x0, x19 0.00 : ffff8000101c4548: blr x2 : __read_once_size(): 0.00 : ffff8000101c454c: ldr x2, [x19, #176] : perf_adjust_freq_unthr_context(): : delta = now - hwc->freq_count_stamp; 0.00 : ffff8000101c4550: ldr x0, [x28, #184] : hwc->freq_count_stamp = now; 0.00 : ffff8000101c4554: str x2, [x28, #184] : delta = now - hwc->freq_count_stamp; 0.00 : ffff8000101c4558: sub x2, x2, x0 : if (delta > 0) 0.00 : ffff8000101c455c: cmp x2, #0x0 0.00 : ffff8000101c4560: b.gt ffff8000101c45fc : event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 0.00 : ffff8000101c4564: ldr x0, [x19, #152] 0.00 : ffff8000101c4568: mov w1, #0x0 // #0 0.00 : ffff8000101c456c: ldr x2, [x0, #152] 0.00 : ffff8000101c4570: mov x0, x19 0.00 : ffff8000101c4574: blr x2 0.00 : ffff8000101c4578: ldr x0, [x19, #152] : perf_pmu_enable(event->pmu); 0.00 : ffff8000101c457c: bl ffff8000101c24b0 0.00 : ffff8000101c4580: ldr x28, [x29, #88] : __read_once_size(): 0.00 : ffff8000101c4584: ldr x19, [x19] : perf_adjust_freq_unthr_context(): : list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 0.00 : ffff8000101c4588: cmp x19, x26 0.00 : ffff8000101c458c: b.ne ffff8000101c4454 // b.any : perf_pmu_enable(ctx->pmu); 0.00 : ffff8000101c4590: ldr x0, [x21] 0.00 : ffff8000101c4594: mov x21, x20 0.00 : ffff8000101c4598: bl ffff8000101c24b0 : raw_spin_unlock(&ctx->lock); 0.00 : ffff8000101c459c: mov x0, x27 0.00 : ffff8000101c45a0: bl ffff800010cd7c38 <_raw_spin_unlock> : perf_event_task_tick(): : list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 0.00 : ffff8000101c45a4: mov x0, x20 0.00 : ffff8000101c45a8: ldr x20, [x0, #48]! 0.00 : ffff8000101c45ac: sub x20, x20, #0x30 0.00 : ffff8000101c45b0: cmp x22, x0 0.00 : ffff8000101c45b4: b.ne ffff8000101c4414 // b.any 0.00 : ffff8000101c45b8: ldr x19, [x29, #16] 0.00 : ffff8000101c45bc: ldp x24, x25, [x29, #56] 0.00 : ffff8000101c45c0: ldp x26, x27, [x29, #72] : perf_adjust_freq_unthr_context(ctx, throttled); : } 0.00 : ffff8000101c45c4: ldp x20, x21, [sp, #24] 0.00 : ffff8000101c45c8: ldp x22, x23, [sp, #40] 0.00 : ffff8000101c45cc: ldp x29, x30, [sp], #96 0.00 : ffff8000101c45d0: ret : perf_adjust_freq_unthr_context(): : hwc->interrupts = 0; 0.00 : ffff8000101c45d4: str xzr, [x28, #168] : perf_log_throttle(event, 1); 0.00 : ffff8000101c45d8: mov x0, x19 0.00 : ffff8000101c45dc: mov w1, #0x1 // #1 0.00 : ffff8000101c45e0: bl ffff8000101bb3e8 : event->pmu->start(event, 0); 0.00 : ffff8000101c45e4: ldr x2, [x19, #152] 0.00 : ffff8000101c45e8: mov w1, #0x0 // #0 0.00 : ffff8000101c45ec: mov x0, x19 0.00 : ffff8000101c45f0: ldr x2, [x2, #152] 0.00 : ffff8000101c45f4: blr x2 0.00 : ffff8000101c45f8: b ffff8000101c4528 : perf_adjust_period(): : period = perf_calculate_period(event, nsec, count); 0.00 : ffff8000101c45fc: ldr x0, [x19, #240] 0.00 : ffff8000101c4600: mov x1, x25 0.00 : ffff8000101c4604: bl ffff8000101bd408 : delta = (s64)(period - hwc->sample_period); 0.00 : ffff8000101c4608: ldr x1, [x28, #136] 0.00 : ffff8000101c460c: sub x0, x0, x1 : delta = (delta + 7) / 8; /* low pass filter */ 0.00 : ffff8000101c4610: adds x2, x0, #0x7 0.00 : ffff8000101c4614: add x0, x0, #0xe 0.00 : ffff8000101c4618: csel x0, x0, x2, mi // mi = first : if (!sample_period) 0.00 : ffff8000101c461c: adds x0, x1, x0, asr #3 0.00 : ffff8000101c4620: lsl x1, x0, #3 0.00 : ffff8000101c4624: b.ne ffff8000101c4630 // b.any 0.00 : ffff8000101c4628: mov x1, #0x8 // #8 0.00 : ffff8000101c462c: mov x0, #0x1 // #1 : hwc->sample_period = sample_period; 0.00 : ffff8000101c4630: str x0, [x28, #136] : __read_once_size(): 0.00 : ffff8000101c4634: ldr x0, [x19, #504] : perf_adjust_period(): : if (local64_read(&hwc->period_left) > 8*sample_period) { 0.00 : ffff8000101c4638: cmp x0, x1 0.00 : ffff8000101c463c: b.le ffff8000101c4644 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101c4640: str xzr, [x19, #504] : perf_adjust_freq_unthr_context(): : event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 0.00 : ffff8000101c4644: ldr x0, [x19, #152] 0.00 : ffff8000101c4648: mov w1, #0x2 // #2 0.00 : ffff8000101c464c: ldr x2, [x0, #152] 0.00 : ffff8000101c4650: b ffff8000101c4570 : pmu_filter_match(): : for_each_sibling_event(sibling, event) { 0.00 : ffff8000101c4654: ldr x28, [x19, #16] 0.00 : ffff8000101c4658: sub x28, x28, #0x10 0.00 : ffff8000101c465c: cmp x19, x28 0.00 : ffff8000101c4660: b.ne ffff8000101c4678 // b.any 0.00 : ffff8000101c4664: b ffff8000101c4510 0.00 : ffff8000101c4668: ldr x28, [x28, #16] 0.00 : ffff8000101c466c: sub x28, x28, #0x10 0.00 : ffff8000101c4670: cmp x19, x28 0.00 : ffff8000101c4674: b.eq ffff8000101c4510 // b.none : __pmu_filter_match(): : return pmu->filter_match ? pmu->filter_match(event) : 1; 0.00 : ffff8000101c4678: ldr x1, [x28, #152] 0.00 : ffff8000101c467c: mov x0, x28 0.00 : ffff8000101c4680: ldr x1, [x1, #280] 0.00 : ffff8000101c4684: cbz x1, ffff8000101c4668 0.00 : ffff8000101c4688: blr x1 : pmu_filter_match(): : if (!__pmu_filter_match(sibling)) 0.00 : ffff8000101c468c: cbnz w0, ffff8000101c4668 : perf_adjust_freq_unthr_context(): : perf_pmu_enable(event->pmu); 0.00 : ffff8000101c4690: ldr x28, [x29, #88] 0.00 : ffff8000101c4694: b ffff8000101c4584 Percent | Source code & Disassembly of vmlinux for cycles (8 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010118200 : : scheduler_tick(): : /* : * This function gets called by the timer code, with HZ frequency. : * We call it with interrupts disabled. : */ : void scheduler_tick(void) : { 0.00 : ffff800010118200: stp x29, x30, [sp, #-64]! : int cpu = smp_processor_id(); 0.00 : ffff800010118204: adrp x0, ffff80001151d000 0.00 : ffff800010118208: add x0, x0, #0x18 : { 0.00 : ffff80001011820c: mov x29, sp 0.00 : ffff800010118210: stp x19, x20, [sp, #16] : struct rq *rq = cpu_rq(cpu); 0.00 : ffff800010118214: adrp x20, ffff800011528000 : { 0.00 : ffff800010118218: stp x21, x22, [sp, #32] : struct rq *rq = cpu_rq(cpu); 0.00 : ffff80001011821c: adrp x21, ffff800011909000 : { 12.76 : ffff800010118220: str x23, [sp, #48] : struct rq *rq = cpu_rq(cpu); 0.00 : ffff800010118224: add x19, x20, #0xe80 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010118228: mrs x1, tpidr_el1 : scheduler_tick(): 0.00 : ffff80001011822c: ldrsw x22, [x0, x1] 0.00 : ffff800010118230: add x0, x21, #0x928 11.95 : ffff800010118234: ldr x0, [x0, x22, lsl #3] 0.00 : ffff800010118238: add x19, x19, x0 : rq_lock(): : : static inline void : rq_lock(struct rq *rq, struct rq_flags *rf) : __acquires(rq->lock) : { : raw_spin_lock(&rq->lock); 0.00 : ffff80001011823c: mov x0, x19 : scheduler_tick(): : struct task_struct *curr = rq->curr; 25.72 : ffff800010118240: ldr x23, [x19, #2352] : rq_lock(): 0.00 : ffff800010118244: bl ffff800010cd80f0 <_raw_spin_lock> : update_rq_clock(): : if (rq->clock_update_flags & RQCF_ACT_SKIP) 0.00 : ffff800010118248: ldr w0, [x19, #2392] 0.00 : ffff80001011824c: tbz w0, #1, ffff8000101182f4 : scheduler_tick(): : sched_clock_tick(); : : rq_lock(rq, &rf); : : update_rq_clock(rq); : curr->sched_class->task_tick(rq, curr, 0); 0.00 : ffff800010118250: ldr x3, [x23, #128] 0.00 : ffff800010118254: mov x1, x23 0.00 : ffff800010118258: mov w2, #0x0 // #0 0.00 : ffff80001011825c: mov x0, x19 : idle_cpu(): : * : * Return: 1 if the CPU is currently idle. 0 otherwise. : */ : int idle_cpu(int cpu) : { : struct rq *rq = cpu_rq(cpu); 0.00 : ffff800010118260: add x21, x21, #0x928 : scheduler_tick(): : curr->sched_class->task_tick(rq, curr, 0); 0.00 : ffff800010118264: ldr x3, [x3, #128] 0.00 : ffff800010118268: blr x3 : calc_global_load_tick(rq); 0.00 : ffff80001011826c: mov x0, x19 0.00 : ffff800010118270: bl ffff80001011b050 : rq_unlock(): : static inline void : rq_unlock(struct rq *rq, struct rq_flags *rf) : __releases(rq->lock) : { : rq_unpin_lock(rq, rf); : raw_spin_unlock(&rq->lock); 0.00 : ffff800010118274: mov x0, x19 0.00 : ffff800010118278: bl ffff800010cd7c38 <_raw_spin_unlock> : scheduler_tick(): : perf_event_task_tick(); 0.00 : ffff80001011827c: bl ffff8000101c4380 : idle_cpu(): : struct rq *rq = cpu_rq(cpu); 0.00 : ffff800010118280: ldr x2, [x21, x22, lsl #3] 0.00 : ffff800010118284: add x0, x20, #0xe80 0.00 : ffff800010118288: add x0, x0, x2 : : if (rq->curr != rq->idle) 0.00 : ffff80001011828c: mov w1, #0x0 // #0 0.00 : ffff800010118290: ldr x3, [x0, #2352] 12.12 : ffff800010118294: ldr x2, [x0, #2360] 0.00 : ffff800010118298: cmp x3, x2 0.00 : ffff80001011829c: b.eq ffff8000101182c0 // b.none : scheduler_tick(): : rq->idle_balance = idle_cpu(cpu); 0.00 : ffff8000101182a0: strb w1, [x19, #2504] : trigger_load_balance(rq); 0.00 : ffff8000101182a4: mov x0, x19 0.00 : ffff8000101182a8: bl ffff800010127d18 : } 0.00 : ffff8000101182ac: ldr x23, [sp, #48] 12.41 : ffff8000101182b0: ldp x19, x20, [sp, #16] 25.04 : ffff8000101182b4: ldp x21, x22, [sp, #32] 0.00 : ffff8000101182b8: ldp x29, x30, [sp], #64 0.00 : ffff8000101182bc: ret : idle_cpu(): : return 0; : : if (rq->nr_running) 0.00 : ffff8000101182c0: ldr w2, [x0, #4] 0.00 : ffff8000101182c4: cbnz w2, ffff8000101182a0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101182c8: ldr x0, [x0, #2976] : idle_cpu(): : return 0; : : #ifdef CONFIG_SMP : if (!llist_empty(&rq->wake_list)) 0.00 : ffff8000101182cc: cmp x0, #0x0 : scheduler_tick(): : trigger_load_balance(rq); 0.00 : ffff8000101182d0: mov x0, x19 0.00 : ffff8000101182d4: cset w1, eq // eq = none : rq->idle_balance = idle_cpu(cpu); 0.00 : ffff8000101182d8: strb w1, [x19, #2504] : trigger_load_balance(rq); 0.00 : ffff8000101182dc: bl ffff800010127d18 : } 0.00 : ffff8000101182e0: ldr x23, [sp, #48] 0.00 : ffff8000101182e4: ldp x19, x20, [sp, #16] 0.00 : ffff8000101182e8: ldp x21, x22, [sp, #32] 0.00 : ffff8000101182ec: ldp x29, x30, [sp], #64 0.00 : ffff8000101182f0: ret : update_rq_clock(): 0.00 : ffff8000101182f4: mov x0, x19 0.00 : ffff8000101182f8: bl ffff800010113d48 0.00 : ffff8000101182fc: b ffff800010118250 Percent | Source code & Disassembly of vmlinux for cycles (8 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010117538 : : scheduler_ipi(): : : rq_unlock_irqrestore(rq, &rf); : } : : void scheduler_ipi(void) : { 0.00 : ffff800010117538: stp x29, x30, [sp, #-48]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001011753c: mrs x0, sp_el0 : scheduler_ipi(): 0.00 : ffff800010117540: mov x29, sp 0.00 : ffff800010117544: stp x19, x20, [sp, #16] 0.00 : ffff800010117548: str x21, [sp, #32] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001011754c: ldr x1, [x0] : scheduler_ipi(): : /* : * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting : * TIF_NEED_RESCHED remotely (for the first time) will also send : * this IPI. : */ : preempt_fold_need_resched(); 0.00 : ffff800010117550: tst w1, #0x2 0.00 : ffff800010117554: b.eq ffff80001011755c // b.none : set_preempt_need_resched(): : task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \ : } while (0) : : static inline void set_preempt_need_resched(void) : { : current_thread_info()->preempt.need_resched = 0; 0.00 : ffff800010117558: str wzr, [x0, #20] : scheduler_ipi(): : : if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) 0.00 : ffff80001011755c: adrp x19, ffff800011528000 0.00 : ffff800010117560: add x0, x19, #0xe80 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010117564: mrs x2, tpidr_el1 : scheduler_ipi(): 0.00 : ffff800010117568: add x1, x0, x2 : got_nohz_idle_kick(): : int cpu = smp_processor_id(); 0.00 : ffff80001011756c: adrp x20, ffff80001151d000 0.00 : ffff800010117570: adrp x21, ffff800011909000 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010117574: ldr x1, [x1, #2976] : scheduler_ipi(): : if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) 0.00 : ffff800010117578: cbnz x1, ffff800010117600 : got_nohz_idle_kick(): : int cpu = smp_processor_id(); 0.00 : ffff80001011757c: add x1, x20, #0x18 : if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK)) 0.00 : ffff800010117580: adrp x21, ffff800011909000 0.00 : ffff800010117584: ldrsw x2, [x1, x2] 0.00 : ffff800010117588: add x1, x21, #0x928 0.00 : ffff80001011758c: ldr x1, [x1, x2, lsl #3] 0.00 : ffff800010117590: add x0, x0, x1 : __read_once_size(): 0.00 : ffff800010117594: ldr w2, [x0, #48] : got_nohz_idle_kick(): 0.00 : ffff800010117598: tst x2, #0x3 0.00 : ffff80001011759c: b.eq ffff8000101175cc // b.none : idle_cpu(): : */ : int idle_cpu(int cpu) : { : struct rq *rq = cpu_rq(cpu); : : if (rq->curr != rq->idle) 74.23 : ffff8000101175a0: ldr x3, [x0, #2352] 0.00 : ffff8000101175a4: ldr x2, [x0, #2360] 0.00 : ffff8000101175a8: cmp x3, x2 0.00 : ffff8000101175ac: b.eq ffff8000101175dc // b.none : got_nohz_idle_kick(): : atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); 0.00 : ffff8000101175b0: add x19, x19, #0xe80 0.00 : ffff8000101175b4: add x19, x1, x19 0.00 : ffff8000101175b8: add x1, x19, #0x30 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101175bc: b ffff800010117688 0.00 : ffff8000101175c0: b ffff800010117688 : __lse_atomic_andnot(): : " " #asm_op " %w[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC_OP(andnot, stclr) 0.00 : ffff8000101175c4: mov w0, #0x3 // #3 0.00 : ffff8000101175c8: stclr w0, [x1] : scheduler_ipi(): : } 0.00 : ffff8000101175cc: ldp x19, x20, [sp, #16] 0.00 : ffff8000101175d0: ldr x21, [sp, #32] 0.00 : ffff8000101175d4: ldp x29, x30, [sp], #48 0.00 : ffff8000101175d8: ret : idle_cpu(): : return 0; : : if (rq->nr_running) 0.00 : ffff8000101175dc: ldr w2, [x0, #4] 0.00 : ffff8000101175e0: cbnz w2, ffff8000101175b0 : __read_once_size(): 0.00 : ffff8000101175e4: ldr x0, [x0, #2976] : idle_cpu(): : return 0; : : #ifdef CONFIG_SMP : if (!llist_empty(&rq->wake_list)) 0.00 : ffff8000101175e8: cbnz x0, ffff8000101175b0 : get_current(): 0.00 : ffff8000101175ec: mrs x0, sp_el0 : test_bit(): 0.00 : ffff8000101175f0: ldr x0, [x0] : got_nohz_idle_kick(): : if (idle_cpu(cpu) && !need_resched()) 0.00 : ffff8000101175f4: tst w0, #0x2 0.00 : ffff8000101175f8: b.ne ffff8000101175b0 // b.any 0.00 : ffff8000101175fc: nop : scheduler_ipi(): : irq_enter(); 0.00 : ffff800010117600: bl ffff8000100ecc58 : got_nohz_idle_kick(): : if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK)) 0.00 : ffff800010117604: add x21, x21, #0x928 : scheduler_ipi(): : sched_ttwu_pending(); 0.00 : ffff800010117608: bl ffff800010117318 : got_nohz_idle_kick(): : int cpu = smp_processor_id(); 0.00 : ffff80001011760c: add x0, x20, #0x18 : if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK)) 0.00 : ffff800010117610: add x1, x19, #0xe80 : __my_cpu_offset(): 0.00 : ffff800010117614: mrs x4, tpidr_el1 : got_nohz_idle_kick(): 0.00 : ffff800010117618: ldrsw x2, [x0, x4] 0.00 : ffff80001011761c: mov x0, x1 0.00 : ffff800010117620: ldr x2, [x21, x2, lsl #3] 0.00 : ffff800010117624: add x3, x0, x2 : __read_once_size(): 0.00 : ffff800010117628: ldr w1, [x3, #48] : got_nohz_idle_kick(): 0.00 : ffff80001011762c: tst x1, #0x3 0.00 : ffff800010117630: b.eq ffff800010117660 // b.none : idle_cpu(): : if (rq->curr != rq->idle) 0.00 : ffff800010117634: ldr x5, [x3, #2352] 0.00 : ffff800010117638: ldr x1, [x3, #2360] 0.00 : ffff80001011763c: cmp x5, x1 0.00 : ffff800010117640: b.eq ffff8000101176a4 // b.none : got_nohz_idle_kick(): : atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); 0.00 : ffff800010117644: add x19, x19, #0xe80 0.00 : ffff800010117648: add x19, x2, x19 0.00 : ffff80001011764c: add x1, x19, #0x30 : arch_static_branch_jump(): 0.00 : ffff800010117650: b ffff800010117674 0.00 : ffff800010117654: b ffff800010117674 : __lse_atomic_andnot(): 0.00 : ffff800010117658: mov w0, #0x3 // #3 0.00 : ffff80001011765c: stclr w0, [x1] : scheduler_ipi(): : irq_exit(); 0.00 : ffff800010117660: bl ffff8000100eccc0 : } 0.00 : ffff800010117664: ldp x19, x20, [sp, #16] 0.00 : ffff800010117668: ldr x21, [sp, #32] 0.00 : ffff80001011766c: ldp x29, x30, [sp], #48 0.00 : ffff800010117670: ret : __ll_sc_atomic_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC_OPS(andnot, bic, ) 0.00 : ffff800010117674: mov w0, #0x3 // #3 0.00 : ffff800010117678: add x3, x19, #0x30 0.00 : ffff80001011767c: b ffff80001011ab5c : scheduler_ipi(): : irq_exit(); 0.00 : ffff800010117680: bl ffff8000100eccc0 0.00 : ffff800010117684: b ffff800010117664 : __ll_sc_atomic_andnot(): 0.00 : ffff800010117688: mov w0, #0x3 // #3 0.00 : ffff80001011768c: add x3, x19, #0x30 0.00 : ffff800010117690: b ffff80001011ab74 : scheduler_ipi(): : } 0.00 : ffff800010117694: ldp x19, x20, [sp, #16] 0.00 : ffff800010117698: ldr x21, [sp, #32] 0.00 : ffff80001011769c: ldp x29, x30, [sp], #48 0.00 : ffff8000101176a0: ret : idle_cpu(): : if (rq->nr_running) 0.00 : ffff8000101176a4: ldr w1, [x3, #4] 0.00 : ffff8000101176a8: cbnz w1, ffff800010117644 : __read_once_size(): 0.00 : ffff8000101176ac: ldr x1, [x3, #2976] : idle_cpu(): : if (!llist_empty(&rq->wake_list)) 0.00 : ffff8000101176b0: cbnz x1, ffff800010117644 : get_current(): 11.75 : ffff8000101176b4: mrs x1, sp_el0 : test_bit(): 14.02 : ffff8000101176b8: ldr x1, [x1] : got_nohz_idle_kick(): : if (idle_cpu(cpu) && !need_resched()) 0.00 : ffff8000101176bc: tst w1, #0x2 0.00 : ffff8000101176c0: b.ne ffff800010117644 // b.any : scheduler_ipi(): : this_rq()->idle_balance = 1; 0.00 : ffff8000101176c4: add x1, x0, x4 0.00 : ffff8000101176c8: mov w2, #0x1 // #1 : raise_softirq_irqoff(SCHED_SOFTIRQ); 0.00 : ffff8000101176cc: mov w0, #0x7 // #7 : this_rq()->idle_balance = 1; 0.00 : ffff8000101176d0: strb w2, [x1, #2504] : raise_softirq_irqoff(SCHED_SOFTIRQ); 0.00 : ffff8000101176d4: bl ffff8000100ecd98 : irq_exit(); 0.00 : ffff8000101176d8: bl ffff8000100eccc0 0.00 : ffff8000101176dc: b ffff800010117664 Percent | Source code & Disassembly of vmlinux for cycles (26 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001046a5d0 : : blk_mq_get_tag(): : else : return __sbitmap_queue_get(bt); : } : : unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) : { 11.35 : ffff80001046a5d0: stp x29, x30, [sp, #-144]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 15.74 : ffff80001046a5d4: mrs x5, sp_el0 : blk_mq_get_tag(): 0.00 : ffff80001046a5d8: mov x29, sp 0.00 : ffff80001046a5dc: stp x19, x20, [sp, #16] 0.00 : ffff80001046a5e0: mov x20, x0 3.56 : ffff80001046a5e4: str x22, [sp, #40] 0.00 : ffff80001046a5e8: adrp x22, ffff800011909000 7.88 : ffff80001046a5ec: str x24, [sp, #56] 0.00 : ffff80001046a5f0: add x0, x22, #0x908 : struct blk_mq_tags *tags = blk_mq_tags_from_data(data); : struct sbitmap_queue *bt; : struct sbq_wait_state *ws; : DEFINE_SBQ_WAIT(wait); 0.00 : ffff80001046a5f4: add x2, x29, #0x78 3.95 : ffff80001046a5f8: stp xzr, xzr, [x29, #88] : { 0.00 : ffff80001046a5fc: ldr x1, [x0] 3.89 : ffff80001046a600: str x1, [x29, #136] 0.00 : ffff80001046a604: mov x1, #0x0 // #0 : struct blk_mq_tags *tags = blk_mq_tags_from_data(data); 0.00 : ffff80001046a608: ldr x3, [x20, #32] 0.00 : ffff80001046a60c: ldr w1, [x20, #8] : DEFINE_SBQ_WAIT(wait); 0.00 : ffff80001046a610: adrp x0, ffff800010130000 0.00 : ffff80001046a614: str x5, [x29, #104] 0.00 : ffff80001046a618: add x0, x0, #0x8 : blk_mq_tags_from_data(): : }; : : static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data) : { : if (data->flags & BLK_MQ_REQ_INTERNAL) : return data->hctx->sched_tags; 7.81 : ffff80001046a61c: ldp x4, x19, [x3, #336] : : return data->hctx->tags; 0.00 : ffff80001046a620: tst x1, #0x4 : blk_mq_get_tag(): 7.74 : ffff80001046a624: str x0, [x29, #112] 0.00 : ffff80001046a628: stp x2, x2, [x29, #120] : blk_mq_tags_from_data(): 0.00 : ffff80001046a62c: csel x19, x19, x4, ne // ne = any : blk_mq_get_tag(): : unsigned int tag_offset; : int tag; : : if (data->flags & BLK_MQ_REQ_RESERVED) { 0.00 : ffff80001046a630: tbz w1, #1, ffff80001046a834 : if (unlikely(!tags->nr_reserved_tags)) { 0.00 : ffff80001046a634: ldr w0, [x19, #4] : WARN_ON_ONCE(1); : return BLK_MQ_TAG_FAIL; : } : bt = &tags->breserved_tags; : tag_offset = 0; 0.00 : ffff80001046a638: mov w24, #0x0 // #0 : bt = &tags->breserved_tags; 0.00 : ffff80001046a63c: add x19, x19, #0x50 : if (unlikely(!tags->nr_reserved_tags)) { 0.00 : ffff80001046a640: cbz w0, ffff80001046a89c 0.00 : ffff80001046a644: str x26, [x29, #72] : } else { : bt = &tags->bitmap_tags; : tag_offset = tags->nr_reserved_tags; : } : : tag = __blk_mq_get_tag(data, bt); 0.00 : ffff80001046a648: mov x1, x19 0.00 : ffff80001046a64c: mov x0, x20 0.00 : ffff80001046a650: bl ffff80001046a0c0 <__blk_mq_get_tag> 0.00 : ffff80001046a654: mov w26, w0 : if (tag != -1) 0.00 : ffff80001046a658: cmn w0, #0x1 0.00 : ffff80001046a65c: b.ne ffff80001046a858 // b.any : goto found_tag; : : if (data->flags & BLK_MQ_REQ_NOWAIT) 0.00 : ffff80001046a660: ldr w1, [x20, #8] 0.00 : ffff80001046a664: tbnz w1, #0, ffff80001046a8b8 0.00 : ffff80001046a668: str x21, [x29, #32] 0.00 : ffff80001046a66c: str x23, [x29, #48] 0.00 : ffff80001046a670: str x25, [x29, #64] : return BLK_MQ_TAG_FAIL; : : ws = bt_wait_ptr(bt, data->hctx); 0.00 : ffff80001046a674: ldr x0, [x20, #32] 0.00 : ffff80001046a678: ldr x21, [x19, #40] : bt_wait_ptr(): : void *priv); : : static inline struct sbq_wait_state *bt_wait_ptr(struct sbitmap_queue *bt, : struct blk_mq_hw_ctx *hctx) : { : if (!hctx) 0.00 : ffff80001046a67c: cbz x0, ffff80001046a6bc : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001046a680: ldr w4, [x0, #328] : atomic_cmpxchg(): : #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) : static inline int : atomic_cmpxchg(atomic_t *v, int old, int new) : { : kasan_check_write(v, sizeof(*v)); : return arch_atomic_cmpxchg(v, old, new); 0.00 : ffff80001046a684: add x3, x0, #0x148 : __read_once_size(): 0.00 : ffff80001046a688: ldr w1, [x0, #328] : sbq_wait_ptr(): : static inline struct sbq_wait_state *sbq_wait_ptr(struct sbitmap_queue *sbq, : atomic_t *wait_index) : { : struct sbq_wait_state *ws; : : ws = &sbq->ws[atomic_read(wait_index)]; 0.00 : ffff80001046a68c: sbfiz x4, x4, #6, #32 : sbq_index_inc(): : return (index + 1) & (SBQ_WAIT_QUEUES - 1); 0.00 : ffff80001046a690: add w2, w1, #0x1 : atomic_cmpxchg(): 0.00 : ffff80001046a694: sxtw x0, w1 : sbq_wait_ptr(): : ws = &sbq->ws[atomic_read(wait_index)]; 0.00 : ffff80001046a698: add x21, x21, x4 : sbq_index_inc(): : return (index + 1) & (SBQ_WAIT_QUEUES - 1); 0.00 : ffff80001046a69c: and w2, w2, #0x7 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001046a6a0: b ffff80001046a8a8 0.00 : ffff80001046a6a4: b ffff80001046a8a8 : __lse__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 0.00 : ffff80001046a6a8: mov x0, x3 0.00 : ffff80001046a6ac: mov w4, w1 0.00 : ffff80001046a6b0: casal w4, w2, [x3] 0.00 : ffff80001046a6b4: mov w0, w4 0.00 : ffff80001046a6b8: ldr x0, [x20, #32] : blk_mq_get_tag(): : /* : * We're out of tags on this hardware queue, kick any : * pending IO submits before going to sleep waiting for : * some to complete. : */ : blk_mq_run_hw_queue(data->hctx, false); 0.00 : ffff80001046a6bc: mov w1, #0x0 // #0 0.00 : ffff80001046a6c0: bl ffff800010465418 : blk_mq_get_ctx(): : return __blk_mq_get_ctx(q, raw_smp_processor_id()); 0.00 : ffff80001046a6c4: adrp x25, ffff80001151d000 : __blk_mq_get_ctx(): : return per_cpu_ptr(q->queue_ctx, cpu); 0.00 : ffff80001046a6c8: adrp x23, ffff800011909000 : blk_mq_get_tag(): : : /* : * Retry tag allocation after running the hardware queue, : * as running the queue may also have found completions. : */ : tag = __blk_mq_get_tag(data, bt); 0.00 : ffff80001046a6cc: mov x1, x19 0.00 : ffff80001046a6d0: mov x0, x20 : blk_mq_get_ctx(): : return __blk_mq_get_ctx(q, raw_smp_processor_id()); 0.00 : ffff80001046a6d4: add x25, x25, #0x18 : blk_mq_get_tag(): 0.00 : ffff80001046a6d8: bl ffff80001046a0c0 <__blk_mq_get_tag> : __blk_mq_get_ctx(): : return per_cpu_ptr(q->queue_ctx, cpu); 0.00 : ffff80001046a6dc: add x23, x23, #0x928 : blk_mq_get_tag(): 0.00 : ffff80001046a6e0: mov w26, w0 : if (tag != -1) 0.00 : ffff80001046a6e4: cmn w0, #0x1 0.00 : ffff80001046a6e8: b.ne ffff80001046a814 // b.any 0.00 : ffff80001046a6ec: nop : break; : : sbitmap_prepare_to_wait(bt, ws, &wait, TASK_UNINTERRUPTIBLE); 0.00 : ffff80001046a6f0: add x2, x29, #0x58 0.00 : ffff80001046a6f4: mov w3, #0x2 // #2 0.00 : ffff80001046a6f8: mov x1, x21 0.00 : ffff80001046a6fc: mov x0, x19 0.00 : ffff80001046a700: bl ffff8000104b0e70 : : tag = __blk_mq_get_tag(data, bt); 0.00 : ffff80001046a704: mov x1, x19 0.00 : ffff80001046a708: mov x0, x20 0.00 : ffff80001046a70c: bl ffff80001046a0c0 <__blk_mq_get_tag> 0.00 : ffff80001046a710: mov w26, w0 : if (tag != -1) 0.00 : ffff80001046a714: cmn w0, #0x1 0.00 : ffff80001046a718: b.ne ffff80001046a814 // b.any : break; : : bt_prev = bt; : io_schedule(); 0.00 : ffff80001046a71c: bl ffff800010cd3008 : : sbitmap_finish_wait(bt, ws, &wait); 0.00 : ffff80001046a720: add x2, x29, #0x58 0.00 : ffff80001046a724: mov x1, x21 0.00 : ffff80001046a728: mov x0, x19 0.00 : ffff80001046a72c: bl ffff8000104b0d48 : blk_mq_get_ctx(): : return __blk_mq_get_ctx(q, raw_smp_processor_id()); 0.00 : ffff80001046a730: mov x0, x25 : blk_mq_get_tag(): : : data->ctx = blk_mq_get_ctx(data->q); : data->hctx = blk_mq_map_queue(data->q, data->cmd_flags, 0.00 : ffff80001046a734: ldr w2, [x20, #16] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001046a738: mrs x1, tpidr_el1 : __blk_mq_get_ctx(): : return per_cpu_ptr(q->queue_ctx, cpu); 0.00 : ffff80001046a73c: ldr w3, [x0, x1] : blk_mq_map_queue(): : type = HCTX_TYPE_POLL; 0.00 : ffff80001046a740: mov w1, #0x2 // #2 : blk_mq_get_ctx(): : return __blk_mq_get_ctx(q, raw_smp_processor_id()); 0.00 : ffff80001046a744: ldr x0, [x20] : __blk_mq_get_ctx(): : return per_cpu_ptr(q->queue_ctx, cpu); 0.00 : ffff80001046a748: ldr x3, [x23, x3, lsl #3] 0.00 : ffff80001046a74c: ldr x0, [x0, #56] 0.00 : ffff80001046a750: add x0, x0, x3 : blk_mq_get_tag(): : data->ctx = blk_mq_get_ctx(data->q); 0.00 : ffff80001046a754: str x0, [x20, #24] : blk_mq_map_queue(): : if (flags & REQ_HIPRI) 0.00 : ffff80001046a758: tbnz w2, #25, ffff80001046a768 : else if ((flags & REQ_OP_MASK) == REQ_OP_READ) 0.00 : ffff80001046a75c: and w2, w2, #0xff 0.00 : ffff80001046a760: cmp w2, #0x0 0.00 : ffff80001046a764: cset w1, eq // eq = none : return ctx->hctxs[type]; 0.00 : ffff80001046a768: ubfiz x1, x1, #3, #2 : blk_mq_get_tag(): : data->ctx); : tags = blk_mq_tags_from_data(data); 0.00 : ffff80001046a76c: ldr w2, [x20, #8] : blk_mq_map_queue(): 0.00 : ffff80001046a770: add x0, x0, x1 : blk_mq_tags_from_data(): : return data->hctx->tags; 0.00 : ffff80001046a774: tst x2, #0x4 : blk_mq_map_queue(): : return ctx->hctxs[type]; 0.00 : ffff80001046a778: ldr x0, [x0, #80] : blk_mq_get_tag(): : data->hctx = blk_mq_map_queue(data->q, data->cmd_flags, 0.00 : ffff80001046a77c: str x0, [x20, #32] : blk_mq_tags_from_data(): : return data->hctx->sched_tags; 0.00 : ffff80001046a780: ldp x3, x1, [x0, #336] : return data->hctx->tags; 0.00 : ffff80001046a784: csel x1, x1, x3, ne // ne = any : blk_mq_get_tag(): : if (data->flags & BLK_MQ_REQ_RESERVED) : bt = &tags->breserved_tags; 0.00 : ffff80001046a788: tst x2, #0x2 0.00 : ffff80001046a78c: add x26, x1, #0x50 0.00 : ffff80001046a790: add x1, x1, #0x10 0.00 : ffff80001046a794: csel x26, x1, x26, eq // eq = none : /* : * If destination hw queue is changed, fake wake up on : * previous queue for compensating the wake up miss, so : * other allocations on previous queue won't be starved. : */ : if (bt != bt_prev) 0.00 : ffff80001046a798: cmp x19, x26 0.00 : ffff80001046a79c: b.eq ffff80001046a7ac // b.none : sbitmap_queue_wake_up(bt_prev); 0.00 : ffff80001046a7a0: mov x0, x19 0.00 : ffff80001046a7a4: bl ffff8000104b1028 0.00 : ffff80001046a7a8: ldr x0, [x20, #32] 0.00 : ffff80001046a7ac: ldr x21, [x26, #40] : bt_wait_ptr(): 0.00 : ffff80001046a7b0: cbz x0, ffff80001046a7f0 : __read_once_size(): 0.00 : ffff80001046a7b4: ldr w4, [x0, #328] : atomic_cmpxchg(): 0.00 : ffff80001046a7b8: add x3, x0, #0x148 : __read_once_size(): 0.00 : ffff80001046a7bc: ldr w1, [x0, #328] : sbq_wait_ptr(): : ws = &sbq->ws[atomic_read(wait_index)]; 0.00 : ffff80001046a7c0: sbfiz x4, x4, #6, #32 : sbq_index_inc(): : return (index + 1) & (SBQ_WAIT_QUEUES - 1); 0.00 : ffff80001046a7c4: add w2, w1, #0x1 : atomic_cmpxchg(): 0.00 : ffff80001046a7c8: sxtw x0, w1 : sbq_wait_ptr(): : ws = &sbq->ws[atomic_read(wait_index)]; 0.00 : ffff80001046a7cc: add x21, x21, x4 : sbq_index_inc(): : return (index + 1) & (SBQ_WAIT_QUEUES - 1); 0.00 : ffff80001046a7d0: and w2, w2, #0x7 : arch_static_branch_jump(): 0.00 : ffff80001046a7d4: b ffff80001046a888 0.00 : ffff80001046a7d8: b ffff80001046a888 : __lse__cmpxchg_case_mb_32(): 0.00 : ffff80001046a7dc: mov x0, x3 0.00 : ffff80001046a7e0: mov w4, w1 0.00 : ffff80001046a7e4: casal w4, w2, [x3] 0.00 : ffff80001046a7e8: mov w0, w4 0.00 : ffff80001046a7ec: ldr x0, [x20, #32] : blk_mq_get_tag(): : tag_offset = 0; 0.00 : ffff80001046a7f0: mov x19, x26 : blk_mq_run_hw_queue(data->hctx, false); 0.00 : ffff80001046a7f4: mov w1, #0x0 // #0 0.00 : ffff80001046a7f8: bl ffff800010465418 : tag = __blk_mq_get_tag(data, bt); 0.00 : ffff80001046a7fc: mov x1, x19 0.00 : ffff80001046a800: mov x0, x20 0.00 : ffff80001046a804: bl ffff80001046a0c0 <__blk_mq_get_tag> 0.00 : ffff80001046a808: mov w26, w0 : if (tag != -1) 0.00 : ffff80001046a80c: cmn w0, #0x1 0.00 : ffff80001046a810: b.eq ffff80001046a6f0 // b.none : : ws = bt_wait_ptr(bt, data->hctx); : } while (1); : : sbitmap_finish_wait(bt, ws, &wait); 0.00 : ffff80001046a814: mov x1, x21 0.00 : ffff80001046a818: add x2, x29, #0x58 0.00 : ffff80001046a81c: mov x0, x19 0.00 : ffff80001046a820: bl ffff8000104b0d48 0.00 : ffff80001046a824: ldr x21, [x29, #32] 0.00 : ffff80001046a828: ldr x23, [x29, #48] 0.00 : ffff80001046a82c: ldr x25, [x29, #64] 0.00 : ffff80001046a830: b ffff80001046a858 0.00 : ffff80001046a834: str x26, [x29, #72] : bt = &tags->bitmap_tags; 0.00 : ffff80001046a838: add x19, x19, #0x10 : tag = __blk_mq_get_tag(data, bt); 0.00 : ffff80001046a83c: mov x0, x20 : tag_offset = tags->nr_reserved_tags; 3.91 : ffff80001046a840: ldur w24, [x19, #-12] : tag = __blk_mq_get_tag(data, bt); 0.00 : ffff80001046a844: mov x1, x19 0.00 : ffff80001046a848: bl ffff80001046a0c0 <__blk_mq_get_tag> 3.90 : ffff80001046a84c: mov w26, w0 : if (tag != -1) 0.00 : ffff80001046a850: cmn w0, #0x1 0.00 : ffff80001046a854: b.eq ffff80001046a660 // b.none : : found_tag: : return tag + tag_offset; 0.00 : ffff80001046a858: add w0, w26, w24 0.00 : ffff80001046a85c: ldr x26, [x29, #72] : } 0.00 : ffff80001046a860: add x22, x22, #0x908 7.94 : ffff80001046a864: ldr x2, [x29, #136] 18.77 : ffff80001046a868: ldr x1, [x22] 0.00 : ffff80001046a86c: eor x1, x2, x1 0.00 : ffff80001046a870: cbnz x1, ffff80001046a8c0 0.00 : ffff80001046a874: ldp x19, x20, [sp, #16] 3.56 : ffff80001046a878: ldr x22, [sp, #40] 0.00 : ffff80001046a87c: ldr x24, [sp, #56] 0.00 : ffff80001046a880: ldp x29, x30, [sp], #144 0.00 : ffff80001046a884: ret : __ll_sc__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 0.00 : ffff80001046a888: and x0, x0, #0xffffffff 0.00 : ffff80001046a88c: b ffff80001046af50 0.00 : ffff80001046a890: ldr x0, [x20, #32] : blk_mq_get_tag(): : tag_offset = 0; 0.00 : ffff80001046a894: mov x19, x26 0.00 : ffff80001046a898: b ffff80001046a7f4 : WARN_ON_ONCE(1); 0.00 : ffff80001046a89c: brk #0x800 : return BLK_MQ_TAG_FAIL; 0.00 : ffff80001046a8a0: mov w0, #0xffffffff // #-1 0.00 : ffff80001046a8a4: b ffff80001046a860 : __ll_sc__cmpxchg_case_mb_32(): 0.00 : ffff80001046a8a8: and x0, x0, #0xffffffff 0.00 : ffff80001046a8ac: b ffff80001046af70 0.00 : ffff80001046a8b0: ldr x0, [x20, #32] 0.00 : ffff80001046a8b4: b ffff80001046a6bc 0.00 : ffff80001046a8b8: ldr x26, [x29, #72] 0.00 : ffff80001046a8bc: b ffff80001046a860 0.00 : ffff80001046a8c0: str x21, [x29, #32] 0.00 : ffff80001046a8c4: str x23, [x29, #48] 0.00 : ffff80001046a8c8: stp x25, x26, [x29, #64] : blk_mq_get_tag(): : } 0.00 : ffff80001046a8cc: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (11 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010127330 : : run_rebalance_domains(): : /* : * run_rebalance_domains is triggered when needed from the scheduler tick. : * Also triggered for nohz idle balancing (with nohz_balancing_kick set). : */ : static __latent_entropy void run_rebalance_domains(struct softirq_action *h) : { 18.25 : ffff800010127330: stp x29, x30, [sp, #-32]! : struct rq *this_rq = this_rq(); 0.00 : ffff800010127334: adrp x2, ffff800011528000 0.00 : ffff800010127338: add x2, x2, #0xe80 : nohz_idle_balance(): : if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK)) 0.00 : ffff80001012733c: adrp x1, ffff800011909000 : run_rebalance_domains(): : { 0.00 : ffff800010127340: mov x29, sp 0.00 : ffff800010127344: stp x19, x20, [sp, #16] : nohz_idle_balance(): : if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK)) 0.00 : ffff800010127348: add x1, x1, #0x928 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001012734c: mrs x19, tpidr_el1 : run_rebalance_domains(): : struct rq *this_rq = this_rq(); 0.00 : ffff800010127350: add x19, x2, x19 : nohz_idle_balance(): : int this_cpu = this_rq->cpu; 0.00 : ffff800010127354: ldr w0, [x19, #2568] : run_rebalance_domains(): : enum cpu_idle_type idle = this_rq->idle_balance ? 9.48 : ffff800010127358: ldrb w20, [x19, #2504] : nohz_idle_balance(): : if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK)) 10.32 : ffff80001012735c: ldr x1, [x1, w0, sxtw #3] 0.00 : ffff800010127360: add x2, x2, x1 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010127364: ldr w1, [x2, #48] : nohz_idle_balance(): 0.00 : ffff800010127368: tst x1, #0x3 0.00 : ffff80001012736c: b.eq ffff800010127394 // b.none : if (idle != CPU_IDLE) { 10.62 : ffff800010127370: cbz w20, ffff8000101273b4 : flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); 0.00 : ffff800010127374: add x0, x2, #0x30 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010127378: b ffff8000101273d0 0.00 : ffff80001012737c: b ffff8000101273d0 : __lse_atomic_fetch_andnot(): : ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \ : ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC_FETCH_OPS(andnot, ldclr) 0.00 : ffff800010127380: mov w1, #0x3 // #3 0.00 : ffff800010127384: ldclral w1, w1, [x0] : nohz_idle_balance(): : if (!(flags & NOHZ_KICK_MASK)) 41.42 : ffff800010127388: tst x1, #0x3 0.00 : ffff80001012738c: b.ne ffff8000101273e4 // b.any 0.00 : ffff800010127390: ldr w0, [x19, #2568] : run_rebalance_domains(): : */ : if (nohz_idle_balance(this_rq, idle)) : return; : : /* normal load balance */ : update_blocked_averages(this_rq->cpu); 9.92 : ffff800010127394: bl ffff8000101226f8 : CPU_IDLE : CPU_NOT_IDLE; 0.00 : ffff800010127398: cmp w20, #0x0 : rebalance_domains(this_rq, idle); 0.00 : ffff80001012739c: mov x0, x19 0.00 : ffff8000101273a0: cset w1, eq // eq = none 0.00 : ffff8000101273a4: bl ffff800010126d70 : } 0.00 : ffff8000101273a8: ldp x19, x20, [sp, #16] 0.00 : ffff8000101273ac: ldp x29, x30, [sp], #32 0.00 : ffff8000101273b0: ret : nohz_idle_balance(): : atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); 0.00 : ffff8000101273b4: add x1, x2, #0x30 : arch_static_branch_jump(): 0.00 : ffff8000101273b8: b ffff8000101273fc 0.00 : ffff8000101273bc: b ffff8000101273fc : __lse_atomic_andnot(): : ATOMIC_OP(andnot, stclr) 0.00 : ffff8000101273c0: mov w0, #0x3 // #3 0.00 : ffff8000101273c4: stclr w0, [x1] 0.00 : ffff8000101273c8: ldr w0, [x19, #2568] 0.00 : ffff8000101273cc: b ffff800010127394 : __ll_sc_atomic_fetch_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC_OPS(andnot, bic, ) 0.00 : ffff8000101273d0: mov w0, #0x3 // #3 0.00 : ffff8000101273d4: add x2, x2, #0x30 0.00 : ffff8000101273d8: b ffff800010128b70 : nohz_idle_balance(): : if (!(flags & NOHZ_KICK_MASK)) 0.00 : ffff8000101273dc: tst x1, #0x3 0.00 : ffff8000101273e0: b.eq ffff800010127390 // b.none : _nohz_idle_balance(this_rq, flags, idle); 0.00 : ffff8000101273e4: mov w2, #0x0 // #0 0.00 : ffff8000101273e8: mov x0, x19 0.00 : ffff8000101273ec: bl ffff800010127130 <_nohz_idle_balance> : run_rebalance_domains(): : } 0.00 : ffff8000101273f0: ldp x19, x20, [sp, #16] 0.00 : ffff8000101273f4: ldp x29, x30, [sp], #32 0.00 : ffff8000101273f8: ret : __ll_sc_atomic_andnot(): 0.00 : ffff8000101273fc: mov w0, #0x3 // #3 0.00 : ffff800010127400: add x2, x2, #0x30 0.00 : ffff800010127404: b ffff800010128b8c 0.00 : ffff800010127408: b ffff800010127390 Percent | Source code & Disassembly of vmlinux for cycles (27 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104b0848 : : sbitmap_get(): : : return nr; : } : : int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin) : { 11.31 : ffff8000104b0848: stp x29, x30, [sp, #-80]! : * Unless we're doing round robin tag allocation, just use the : * alloc_hint to find the right word index. No point in looping : * twice in find_next_zero_bit() for that case. : */ : if (round_robin) : alloc_hint = SB_NR_TO_BIT(sb, alloc_hint); 0.00 : ffff8000104b084c: mov w3, #0xffffffff // #-1 0.00 : ffff8000104b0850: ands w2, w2, #0xff : { 0.00 : ffff8000104b0854: mov x29, sp 25.95 : ffff8000104b0858: str x19, [sp, #16] 0.00 : ffff8000104b085c: str x21, [sp, #32] : else : alloc_hint = 0; : : for (i = 0; i < sb->map_nr; i++) { 0.00 : ffff8000104b0860: ldp w4, w5, [x0, #4] : alloc_hint = SB_NR_TO_BIT(sb, alloc_hint); 0.00 : ffff8000104b0864: lsl w21, w3, w4 0.00 : ffff8000104b0868: bic w21, w1, w21 : index = SB_NR_TO_INDEX(sb, alloc_hint); 0.00 : ffff8000104b086c: lsr w19, w1, w4 : alloc_hint = SB_NR_TO_BIT(sb, alloc_hint); 0.00 : ffff8000104b0870: csel w21, w21, wzr, ne // ne = any : for (i = 0; i < sb->map_nr; i++) { 0.00 : ffff8000104b0874: cbz w5, ffff8000104b0928 0.00 : ffff8000104b0878: stp x22, x23, [x29, #40] 0.00 : ffff8000104b087c: eor w22, w2, #0x1 0.00 : ffff8000104b0880: str x26, [x29, #72] 0.00 : ffff8000104b0884: mov x26, x0 7.47 : ffff8000104b0888: str x20, [x29, #24] : __xchg_case_mb_64(): : __XCHG_CASE(w, , rel_, 32, , , , , l, "memory") : __XCHG_CASE( , , rel_, 64, , , , , l, "memory") : __XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory") : __XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory") : __XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory") : __XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory") 0.00 : ffff8000104b088c: mov x23, #0x0 // #0 0.00 : ffff8000104b0890: stp x24, x25, [x29, #56] : sbitmap_get(): 0.00 : ffff8000104b0894: mov w20, #0x0 // #0 : sbitmap_find_bit_in_index(): : nr = __sbitmap_get_word(&sb->map[index].word, 0.00 : ffff8000104b0898: mov w24, #0xc0 // #192 0.00 : ffff8000104b089c: nop 3.75 : ffff8000104b08a0: smull x25, w19, w24 0.00 : ffff8000104b08a4: ldr x1, [x26, #16] 0.00 : ffff8000104b08a8: mov w3, w22 0.00 : ffff8000104b08ac: mov w2, w21 0.00 : ffff8000104b08b0: add x0, x1, x25 7.51 : ffff8000104b08b4: ldr x1, [x1, x25] 0.00 : ffff8000104b08b8: add x0, x0, #0x40 0.00 : ffff8000104b08bc: bl ffff8000104b0720 <__sbitmap_get_word> : if (nr != -1) 0.00 : ffff8000104b08c0: cmn w0, #0x1 0.00 : ffff8000104b08c4: b.ne ffff8000104b09b0 // b.any : sbitmap_deferred_clear(): : spin_lock_irqsave(&sb->map[index].swap_lock, flags); 0.00 : ffff8000104b08c8: ldr x0, [x26, #16] 0.00 : ffff8000104b08cc: add x0, x0, x25 0.00 : ffff8000104b08d0: add x0, x0, #0x88 0.00 : ffff8000104b08d4: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff8000104b08d8: mov x6, x0 : if (!sb->map[index].cleared) 0.00 : ffff8000104b08dc: ldr x0, [x26, #16] 0.00 : ffff8000104b08e0: add x0, x0, x25 0.00 : ffff8000104b08e4: ldr x1, [x0, #128] 0.00 : ffff8000104b08e8: cbnz x1, ffff8000104b093c : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff8000104b08ec: add x0, x0, #0x88 0.00 : ffff8000104b08f0: mov x1, x6 0.00 : ffff8000104b08f4: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : sbitmap_get(): : break; : } : : /* Jump to next index. */ : alloc_hint = 0; : if (++index >= sb->map_nr) 0.00 : ffff8000104b08f8: add w19, w19, #0x1 0.00 : ffff8000104b08fc: ldr w0, [x26, #8] : for (i = 0; i < sb->map_nr; i++) { 0.00 : ffff8000104b0900: add w20, w20, #0x1 : alloc_hint = 0; 0.00 : ffff8000104b0904: mov w21, #0x0 // #0 : index = 0; 0.00 : ffff8000104b0908: cmp w0, w19 0.00 : ffff8000104b090c: csel w19, w19, wzr, hi // hi = pmore : for (i = 0; i < sb->map_nr; i++) { 0.00 : ffff8000104b0910: cmp w0, w20 0.00 : ffff8000104b0914: b.hi ffff8000104b08a0 // b.pmore 0.00 : ffff8000104b0918: ldr x20, [x29, #24] 0.00 : ffff8000104b091c: ldp x22, x23, [x29, #40] 0.00 : ffff8000104b0920: ldp x24, x25, [x29, #56] 0.00 : ffff8000104b0924: ldr x26, [x29, #72] : index = 0; 0.00 : ffff8000104b0928: mov w0, #0xffffffff // #-1 : } : : return nr; : } 0.00 : ffff8000104b092c: ldr x19, [sp, #16] 0.00 : ffff8000104b0930: ldr x21, [sp, #32] 0.00 : ffff8000104b0934: ldp x29, x30, [sp], #80 0.00 : ffff8000104b0938: ret : __xchg_case_mb_64(): 0.00 : ffff8000104b093c: add x0, x0, #0x80 0.00 : ffff8000104b0940: prfm pstl1strm, [x0] 7.11 : ffff8000104b0944: ldxr x5, [x0] 0.00 : ffff8000104b0948: stlxr w1, x23, [x0] 0.00 : ffff8000104b094c: cbnz w1, ffff8000104b0944 0.00 : ffff8000104b0950: dmb ish 0.00 : ffff8000104b0954: mvn x5, x5 : sbitmap_deferred_clear(): : val = sb->map[index].word; 0.00 : ffff8000104b0958: ldr x3, [x26, #16] 0.00 : ffff8000104b095c: add x3, x3, x25 0.00 : ffff8000104b0960: mov x0, x3 0.00 : ffff8000104b0964: ldr x4, [x0, #64]! : } while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val); 0.00 : ffff8000104b0968: and x2, x4, x5 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000104b096c: b ffff8000104b09a4 0.00 : ffff8000104b0970: b ffff8000104b09a4 : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff8000104b0974: mov x1, x4 0.00 : ffff8000104b0978: mov x7, x1 0.00 : ffff8000104b097c: casal x7, x2, [x0] 0.00 : ffff8000104b0980: mov x0, x7 : sbitmap_deferred_clear(): 0.00 : ffff8000104b0984: cmp x4, x0 0.00 : ffff8000104b0988: b.ne ffff8000104b0958 // b.any : spin_unlock_irqrestore(&sb->map[index].swap_lock, flags); 0.00 : ffff8000104b098c: ldr x0, [x26, #16] : spin_unlock_irqrestore(): 0.00 : ffff8000104b0990: mov x1, x6 : sbitmap_deferred_clear(): 0.00 : ffff8000104b0994: add x0, x0, x25 : spin_unlock_irqrestore(): 0.00 : ffff8000104b0998: add x0, x0, #0x88 0.00 : ffff8000104b099c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff8000104b09a0: b ffff8000104b08a4 : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff8000104b09a4: add x3, x3, #0x40 0.00 : ffff8000104b09a8: b ffff8000104b1420 <__sbitmap_queue_get+0x188> 0.00 : ffff8000104b09ac: b ffff8000104b0984 : sbitmap_get(): : nr += index << sb->shift; 0.00 : ffff8000104b09b0: ldr w1, [x26, #4] 0.00 : ffff8000104b09b4: ldr x20, [x29, #24] 0.00 : ffff8000104b09b8: ldp x22, x23, [x29, #40] 0.00 : ffff8000104b09bc: lsl w19, w19, w1 25.64 : ffff8000104b09c0: ldp x24, x25, [x29, #56] 0.00 : ffff8000104b09c4: add w0, w19, w0 11.26 : ffff8000104b09c8: ldr x26, [x29, #72] : } 0.00 : ffff8000104b09cc: ldr x19, [sp, #16] 0.00 : ffff8000104b09d0: ldr x21, [sp, #32] 0.00 : ffff8000104b09d4: ldp x29, x30, [sp], #80 0.00 : ffff8000104b09d8: ret Percent | Source code & Disassembly of vmlinux for cycles (13 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010134e10 : : update_dl_rq_load_avg(): : * runnable_load_sum = load_sum : * : */ : : int update_dl_rq_load_avg(u64 now, struct rq *rq, int running) : { 0.00 : ffff800010134e10: mov w9, w2 : ___update_load_sum(): : delta = now - sa->last_update_time; 0.00 : ffff800010134e14: ldr x2, [x1, #2688] : update_dl_rq_load_avg(): : { 0.00 : ffff800010134e18: mov x8, x1 : ___update_load_sum(): : delta = now - sa->last_update_time; 0.00 : ffff800010134e1c: add x7, x1, #0xa80 : if ((s64)delta < 0) { 0.00 : ffff800010134e20: subs x1, x0, x2 27.06 : ffff800010134e24: b.mi ffff800010134f28 // b.first : delta >>= 10; 0.00 : ffff800010134e28: lsr x3, x1, #10 : if (!delta) 0.00 : ffff800010134e2c: cbz x3, ffff800010134f20 : update_dl_rq_load_avg(): : { 0.00 : ffff800010134e30: stp x29, x30, [sp, #-16]! : ___update_load_sum(): : sa->last_update_time += delta << 10; 0.00 : ffff800010134e34: and x1, x1, #0xfffffffffffffc00 0.00 : ffff800010134e38: add x1, x1, x2 0.00 : ffff800010134e3c: mov w0, w3 : update_dl_rq_load_avg(): : { 0.00 : ffff800010134e40: mov x29, sp : ___update_load_sum(): : sa->last_update_time += delta << 10; 0.00 : ffff800010134e44: str x1, [x8, #2688] : update_dl_rq_load_avg(): : if (___update_load_sum(now, &rq->avg_dl, 0.00 : ffff800010134e48: sxtw x11, w9 0.00 : ffff800010134e4c: ldr w1, [x7, #28] 0.00 : ffff800010134e50: add x3, x3, w1, uxtw 0.00 : ffff800010134e54: lsr x10, x3, #10 : ___update_load_sum(): : if (!load) 0.00 : ffff800010134e58: cbnz x11, ffff800010134f34 : runnable = running = 0; 0.00 : ffff800010134e5c: mov x13, #0x0 // #0 0.00 : ffff800010134e60: mov w9, #0x0 // #0 : accumulate_sum(): : if (periods) { 0.00 : ffff800010134e64: cbz x10, ffff800010134f58 : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.00 : ffff800010134e68: cmp x10, #0x7e0 : accumulate_sum(): : sa->load_sum = decay_load(sa->load_sum, periods); 8.24 : ffff800010134e6c: ldr x4, [x7, #8] : decay_load(): : if (unlikely(n > LOAD_AVG_PERIOD * 63)) 0.00 : ffff800010134e70: b.hi ffff800010134f8c // b.pmore : if (unlikely(local_n >= LOAD_AVG_PERIOD)) { 7.21 : ffff800010134e74: cmp x10, #0x1f 0.00 : ffff800010134e78: ldr w12, [x7, #24] 0.00 : ffff800010134e7c: ldr x5, [x7, #16] 0.00 : ffff800010134e80: b.hi ffff800010134f98 // b.pmore : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010134e84: adrp x6, ffff800010d06000 0.00 : ffff800010134e88: add x6, x6, #0x450 : mul_u64_u32_shr(): : #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) : : #ifndef mul_u64_u32_shr : static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) : { : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff800010134e8c: ldr w2, [x6, w10, uxtw #2] 0.00 : ffff800010134e90: mul x14, x2, x4 0.00 : ffff800010134e94: mul x6, x2, x5 0.00 : ffff800010134e98: umulh x4, x2, x4 16.62 : ffff800010134e9c: umulh x5, x2, x5 0.00 : ffff800010134ea0: extr x4, x4, x14, #32 0.00 : ffff800010134ea4: extr x5, x5, x6, #32 0.00 : ffff800010134ea8: stp x4, x5, [x7, #8] 0.00 : ffff800010134eac: mul x2, x12, x2 0.00 : ffff800010134eb0: lsr x2, x2, #32 : accumulate_sum(): : sa->util_sum = decay_load((u64)(sa->util_sum), periods); 0.00 : ffff800010134eb4: str w2, [x7, #24] : delta %= 1024; 0.00 : ffff800010134eb8: and x14, x3, #0x3ff 0.00 : ffff800010134ebc: mov w12, w14 : if (load) { 0.00 : ffff800010134ec0: cbnz x11, ffff800010134f70 : sa->period_contrib = delta; 0.00 : ffff800010134ec4: str w14, [x7, #28] : if (runnable) 0.00 : ffff800010134ec8: cbz x13, ffff800010134edc : sa->runnable_load_sum += runnable * contrib; 0.00 : ffff800010134ecc: ldr x2, [x7, #16] 0.00 : ffff800010134ed0: mov w1, w0 0.00 : ffff800010134ed4: madd x13, x1, x13, x2 0.00 : ffff800010134ed8: str x13, [x7, #16] : if (running) 0.00 : ffff800010134edc: cbz w9, ffff800010134eec : sa->util_sum += contrib << SCHED_CAPACITY_SHIFT; 0.00 : ffff800010134ee0: ldr w1, [x7, #24] 0.00 : ffff800010134ee4: add w0, w1, w0, lsl #10 0.00 : ffff800010134ee8: str w0, [x7, #24] : ___update_load_sum(): : if (!accumulate_sum(delta, sa, load, runnable, running)) 0.00 : ffff800010134eec: cbz w10, ffff800010134f5c : div_u64_rem(): : return dividend / divisor; 0.00 : ffff800010134ef0: ldp x4, x3, [x7, #8] : ___update_load_avg(): : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010134ef4: mov w0, #0xb67e // #46718 : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 31.34 : ffff800010134ef8: ldr w2, [x7, #24] : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010134efc: add w5, w12, w0 : update_dl_rq_load_avg(): : running, : running)) { : : ___update_load_avg(&rq->avg_dl, 1, 1); : trace_pelt_dl_tp(rq); : return 1; 0.00 : ffff800010134f00: mov w0, #0x1 // #1 : div_u64_rem(): 0.00 : ffff800010134f04: udiv x4, x4, x5 0.00 : ffff800010134f08: udiv x3, x3, x5 : ___update_load_avg(): : WRITE_ONCE(sa->util_avg, sa->util_sum / divider); 0.00 : ffff800010134f0c: udiv w1, w2, w5 : sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider); 0.00 : ffff800010134f10: stp x4, x3, [x7, #32] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010134f14: str x1, [x8, #2736] : update_dl_rq_load_avg(): : } : : return 0; : } 0.00 : ffff800010134f18: ldp x29, x30, [sp], #16 0.00 : ffff800010134f1c: ret : return 0; 0.00 : ffff800010134f20: mov w0, #0x0 // #0 : } 0.00 : ffff800010134f24: ret : ___update_load_sum(): : sa->last_update_time = now; 0.00 : ffff800010134f28: str x0, [x8, #2688] : update_dl_rq_load_avg(): : return 0; 0.00 : ffff800010134f2c: mov w0, #0x0 // #0 0.00 : ffff800010134f30: ret : accumulate_sum(): : if (periods) { 0.00 : ffff800010134f34: cbnz x10, ffff800010134f68 : sa->period_contrib = delta; 0.00 : ffff800010134f38: mov w12, w3 0.00 : ffff800010134f3c: mov x13, x11 0.00 : ffff800010134f40: str w3, [x7, #28] : sa->load_sum += load * contrib; 0.00 : ffff800010134f44: ldr x2, [x7, #8] 0.00 : ffff800010134f48: mov w1, w0 0.00 : ffff800010134f4c: madd x11, x1, x11, x2 0.00 : ffff800010134f50: str x11, [x7, #8] 0.00 : ffff800010134f54: b ffff800010134ec8 : sa->period_contrib = delta; 0.00 : ffff800010134f58: str w3, [x7, #28] : update_dl_rq_load_avg(): : return 0; 0.00 : ffff800010134f5c: mov w0, #0x0 // #0 : } 9.54 : ffff800010134f60: ldp x29, x30, [sp], #16 0.00 : ffff800010134f64: ret : accumulate_sum(): : if (periods) { 0.00 : ffff800010134f68: mov x13, x11 0.00 : ffff800010134f6c: b ffff800010134e68 : contrib = __accumulate_pelt_segments(periods, 0.00 : ffff800010134f70: mov w0, #0x400 // #1024 0.00 : ffff800010134f74: mov w2, w14 0.00 : ffff800010134f78: sub w1, w0, w1 0.00 : ffff800010134f7c: mov x0, x10 0.00 : ffff800010134f80: bl ffff8000101346b8 <__accumulate_pelt_segments> : sa->period_contrib = delta; 0.00 : ffff800010134f84: str w14, [x7, #28] 0.00 : ffff800010134f88: b ffff800010134f44 : sa->runnable_load_sum = 0.00 : ffff800010134f8c: mov w2, #0x0 // #0 0.00 : ffff800010134f90: stp xzr, xzr, [x7, #8] 0.00 : ffff800010134f94: b ffff800010134eb4 : decay_load(): : val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); 0.00 : ffff800010134f98: and x14, x10, #0x1f 0.00 : ffff800010134f9c: adrp x2, ffff800010d06000 0.00 : ffff800010134fa0: add x2, x2, #0x450 : val >>= local_n / LOAD_AVG_PERIOD; 0.00 : ffff800010134fa4: lsr w6, w10, #5 0.00 : ffff800010134fa8: lsr x4, x4, x6 : mul_u64_u32_shr(): : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff800010134fac: ldr w2, [x2, x14, lsl #2] : decay_load(): 0.00 : ffff800010134fb0: lsr x5, x5, x6 0.00 : ffff800010134fb4: lsr x12, x12, x6 : mul_u64_u32_shr(): 0.00 : ffff800010134fb8: mul x14, x4, x2 0.00 : ffff800010134fbc: umulh x4, x4, x2 0.00 : ffff800010134fc0: mul x6, x5, x2 0.00 : ffff800010134fc4: umulh x5, x5, x2 0.00 : ffff800010134fc8: extr x4, x4, x14, #32 0.00 : ffff800010134fcc: str x4, [x7, #8] 0.00 : ffff800010134fd0: extr x4, x5, x6, #32 0.00 : ffff800010134fd4: str x4, [x7, #16] 0.00 : ffff800010134fd8: b ffff800010134eac Percent | Source code & Disassembly of vmlinux for cycles (21 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cb7700 <__arch_copy_to_user>: : __arch_copy_to_user(): : uao_stp 9998f, \ptr, \regB, \regC, \val : .endm : : end .req x5 : SYM_FUNC_START(__arch_copy_to_user) : add end, x0, x2 0.00 : ffff800010cb7700: add x5, x0, x2 : C_l .req x11 : C_h .req x12 : D_l .req x13 : D_h .req x14 : : mov dst, dstin 0.00 : ffff800010cb7704: mov x6, x0 : cmp count, #16 0.00 : ffff800010cb7708: cmp x2, #0x10 : /*When memory length is less than 16, the accessed are not aligned.*/ : b.lo .Ltiny15 0.00 : ffff800010cb770c: b.cc ffff800010cb77ac <__arch_copy_to_user+0xac> // b.lo, b.ul, b.last : : neg tmp2, src 0.00 : ffff800010cb7710: neg x4, x1 : ands tmp2, tmp2, #15/* Bytes to reach alignment. */ 0.00 : ffff800010cb7714: ands x4, x4, #0xf : b.eq .LSrcAligned 0.00 : ffff800010cb7718: b.eq ffff800010cb7760 <__arch_copy_to_user+0x60> // b.none : sub count, count, tmp2 0.00 : ffff800010cb771c: sub x2, x2, x4 : * Copy the leading memory data from src to dst in an increasing : * address order.By this way,the risk of overwriting the source : * memory data is eliminated when the distance between src and : * dst is less than 16. The memory accesses here are alignment. : */ : tbz tmp2, #0, 1f 0.00 : ffff800010cb7720: tbz w4, #0, ffff800010cb7730 <__arch_copy_to_user+0x30> : ldrb1 tmp1w, src, #1 0.00 : ffff800010cb7724: ldrb w3, [x1], #1 : strb1 tmp1w, dst, #1 0.00 : ffff800010cb7728: strb w3, [x6], #1 0.00 : ffff800010cb772c: nop : 1: : tbz tmp2, #1, 2f 0.00 : ffff800010cb7730: tbz w4, #1, ffff800010cb7740 <__arch_copy_to_user+0x40> : ldrh1 tmp1w, src, #2 0.00 : ffff800010cb7734: ldrh w3, [x1], #2 : strh1 tmp1w, dst, #2 0.00 : ffff800010cb7738: strh w3, [x6], #2 0.00 : ffff800010cb773c: nop : 2: : tbz tmp2, #2, 3f 0.00 : ffff800010cb7740: tbz w4, #2, ffff800010cb7750 <__arch_copy_to_user+0x50> : ldr1 tmp1w, src, #4 0.00 : ffff800010cb7744: ldr w3, [x1], #4 : str1 tmp1w, dst, #4 0.00 : ffff800010cb7748: str w3, [x6], #4 0.00 : ffff800010cb774c: nop : 3: : tbz tmp2, #3, .LSrcAligned 0.00 : ffff800010cb7750: tbz w4, #3, ffff800010cb7760 <__arch_copy_to_user+0x60> : ldr1 tmp1, src, #8 0.00 : ffff800010cb7754: ldr x3, [x1], #8 : str1 tmp1, dst, #8 0.00 : ffff800010cb7758: str x3, [x6], #8 0.00 : ffff800010cb775c: nop : : .LSrcAligned: : cmp count, #64 4.88 : ffff800010cb7760: cmp x2, #0x40 : b.ge .Lcpy_over64 0.00 : ffff800010cb7764: b.ge ffff800010cb77f0 <__arch_copy_to_user+0xf0> // b.tcont : .Ltail63: : /* : * Copy up to 48 bytes of data. At this point we only need the : * bottom 6 bits of count to be accurate. : */ : ands tmp1, count, #0x30 14.18 : ffff800010cb7768: ands x3, x2, #0x30 : b.eq .Ltiny15 0.00 : ffff800010cb776c: b.eq ffff800010cb77ac <__arch_copy_to_user+0xac> // b.none : cmp tmp1w, #0x20 4.96 : ffff800010cb7770: cmp w3, #0x20 : b.eq 1f 0.00 : ffff800010cb7774: b.eq ffff800010cb778c <__arch_copy_to_user+0x8c> // b.none : b.lt 2f 0.00 : ffff800010cb7778: b.lt ffff800010cb779c <__arch_copy_to_user+0x9c> // b.tstop : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb777c: ldp x7, x8, [x1], #16 : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb7780: stp x7, x8, [x6], #16 0.00 : ffff800010cb7784: nop 0.00 : ffff800010cb7788: nop : 1: : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb778c: ldp x7, x8, [x1], #16 : stp1 A_l, A_h, dst, #16 42.11 : ffff800010cb7790: stp x7, x8, [x6], #16 0.00 : ffff800010cb7794: nop 0.00 : ffff800010cb7798: nop : 2: : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb779c: ldp x7, x8, [x1], #16 : stp1 A_l, A_h, dst, #16 24.05 : ffff800010cb77a0: stp x7, x8, [x6], #16 4.95 : ffff800010cb77a4: nop 0.00 : ffff800010cb77a8: nop : * precondition that src address is at least 16 bytes bigger than dst : * address,otherwise some source data will be overwritten when memove : * call memcpy directly. To make memmove simpler and decouple the : * memcpy's dependency on memmove, withdrew the original process. : */ : tbz count, #3, 1f 0.00 : ffff800010cb77ac: tbz w2, #3, ffff800010cb77bc <__arch_copy_to_user+0xbc> : ldr1 tmp1, src, #8 0.00 : ffff800010cb77b0: ldr x3, [x1], #8 : str1 tmp1, dst, #8 0.00 : ffff800010cb77b4: str x3, [x6], #8 0.00 : ffff800010cb77b8: nop : 1: : tbz count, #2, 2f 0.00 : ffff800010cb77bc: tbz w2, #2, ffff800010cb77cc <__arch_copy_to_user+0xcc> : ldr1 tmp1w, src, #4 0.00 : ffff800010cb77c0: ldr w3, [x1], #4 : str1 tmp1w, dst, #4 0.00 : ffff800010cb77c4: str w3, [x6], #4 0.00 : ffff800010cb77c8: nop : 2: : tbz count, #1, 3f 0.00 : ffff800010cb77cc: tbz w2, #1, ffff800010cb77dc <__arch_copy_to_user+0xdc> : ldrh1 tmp1w, src, #2 0.00 : ffff800010cb77d0: ldrh w3, [x1], #2 : strh1 tmp1w, dst, #2 0.00 : ffff800010cb77d4: strh w3, [x6], #2 0.00 : ffff800010cb77d8: nop : 3: : tbz count, #0, .Lexitfunc 4.86 : ffff800010cb77dc: tbz w2, #0, ffff800010cb7910 <__arch_copy_to_user+0x210> : ldrb1 tmp1w, src, #1 0.00 : ffff800010cb77e0: ldrb w3, [x1], #1 : strb1 tmp1w, dst, #1 0.00 : ffff800010cb77e4: strb w3, [x6], #1 0.00 : ffff800010cb77e8: nop : : b .Lexitfunc 0.00 : ffff800010cb77ec: b ffff800010cb7910 <__arch_copy_to_user+0x210> : : .Lcpy_over64: : subs count, count, #128 0.00 : ffff800010cb77f0: subs x2, x2, #0x80 : b.ge .Lcpy_body_large 0.00 : ffff800010cb77f4: b.ge ffff800010cb7880 <__arch_copy_to_user+0x180> // b.tcont : /* : * Less than 128 bytes to copy, so handle 64 here and then jump : * to the tail. : */ : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb77f8: ldp x7, x8, [x1], #16 : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb77fc: stp x7, x8, [x6], #16 0.00 : ffff800010cb7800: nop 0.00 : ffff800010cb7804: nop : ldp1 B_l, B_h, src, #16 0.00 : ffff800010cb7808: ldp x9, x10, [x1], #16 : ldp1 C_l, C_h, src, #16 0.00 : ffff800010cb780c: ldp x11, x12, [x1], #16 : stp1 B_l, B_h, dst, #16 0.00 : ffff800010cb7810: stp x9, x10, [x6], #16 0.00 : ffff800010cb7814: nop 0.00 : ffff800010cb7818: nop : stp1 C_l, C_h, dst, #16 0.00 : ffff800010cb781c: stp x11, x12, [x6], #16 0.00 : ffff800010cb7820: nop 0.00 : ffff800010cb7824: nop : ldp1 D_l, D_h, src, #16 0.00 : ffff800010cb7828: ldp x13, x14, [x1], #16 : stp1 D_l, D_h, dst, #16 0.00 : ffff800010cb782c: stp x13, x14, [x6], #16 0.00 : ffff800010cb7830: nop 0.00 : ffff800010cb7834: nop : : tst count, #0x3f 0.00 : ffff800010cb7838: tst x2, #0x3f : b.ne .Ltail63 0.00 : ffff800010cb783c: b.ne ffff800010cb7768 <__arch_copy_to_user+0x68> // b.any : b .Lexitfunc 0.00 : ffff800010cb7840: b ffff800010cb7910 <__arch_copy_to_user+0x210> 0.00 : ffff800010cb7844: nop 0.00 : ffff800010cb7848: nop 0.00 : ffff800010cb784c: nop 0.00 : ffff800010cb7850: nop 0.00 : ffff800010cb7854: nop 0.00 : ffff800010cb7858: nop 0.00 : ffff800010cb785c: nop 0.00 : ffff800010cb7860: nop 0.00 : ffff800010cb7864: nop 0.00 : ffff800010cb7868: nop 0.00 : ffff800010cb786c: nop 0.00 : ffff800010cb7870: nop 0.00 : ffff800010cb7874: nop 0.00 : ffff800010cb7878: nop 0.00 : ffff800010cb787c: nop : * 64 bytes per line this ensures the entire loop is in one line. : */ : .p2align L1_CACHE_SHIFT : .Lcpy_body_large: : /* pre-get 64 bytes data. */ : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb7880: ldp x7, x8, [x1], #16 : ldp1 B_l, B_h, src, #16 0.00 : ffff800010cb7884: ldp x9, x10, [x1], #16 : ldp1 C_l, C_h, src, #16 0.00 : ffff800010cb7888: ldp x11, x12, [x1], #16 : ldp1 D_l, D_h, src, #16 0.00 : ffff800010cb788c: ldp x13, x14, [x1], #16 : 1: : /* : * interlace the load of next 64 bytes data block with store of the last : * loaded 64 bytes data. : */ : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb7890: stp x7, x8, [x6], #16 0.00 : ffff800010cb7894: nop 0.00 : ffff800010cb7898: nop : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb789c: ldp x7, x8, [x1], #16 : stp1 B_l, B_h, dst, #16 0.00 : ffff800010cb78a0: stp x9, x10, [x6], #16 0.00 : ffff800010cb78a4: nop 0.00 : ffff800010cb78a8: nop : ldp1 B_l, B_h, src, #16 0.00 : ffff800010cb78ac: ldp x9, x10, [x1], #16 : stp1 C_l, C_h, dst, #16 0.00 : ffff800010cb78b0: stp x11, x12, [x6], #16 0.00 : ffff800010cb78b4: nop 0.00 : ffff800010cb78b8: nop : ldp1 C_l, C_h, src, #16 0.00 : ffff800010cb78bc: ldp x11, x12, [x1], #16 : stp1 D_l, D_h, dst, #16 0.00 : ffff800010cb78c0: stp x13, x14, [x6], #16 0.00 : ffff800010cb78c4: nop 0.00 : ffff800010cb78c8: nop : ldp1 D_l, D_h, src, #16 0.00 : ffff800010cb78cc: ldp x13, x14, [x1], #16 : subs count, count, #64 0.00 : ffff800010cb78d0: subs x2, x2, #0x40 : b.ge 1b 0.00 : ffff800010cb78d4: b.ge ffff800010cb7890 <__arch_copy_to_user+0x190> // b.tcont : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb78d8: stp x7, x8, [x6], #16 0.00 : ffff800010cb78dc: nop 0.00 : ffff800010cb78e0: nop : stp1 B_l, B_h, dst, #16 0.00 : ffff800010cb78e4: stp x9, x10, [x6], #16 0.00 : ffff800010cb78e8: nop 0.00 : ffff800010cb78ec: nop : stp1 C_l, C_h, dst, #16 0.00 : ffff800010cb78f0: stp x11, x12, [x6], #16 0.00 : ffff800010cb78f4: nop 0.00 : ffff800010cb78f8: nop : stp1 D_l, D_h, dst, #16 0.00 : ffff800010cb78fc: stp x13, x14, [x6], #16 0.00 : ffff800010cb7900: nop 0.00 : ffff800010cb7904: nop : : tst count, #0x3f 0.00 : ffff800010cb7908: tst x2, #0x3f : b.ne .Ltail63 0.00 : ffff800010cb790c: b.ne ffff800010cb7768 <__arch_copy_to_user+0x68> // b.any : #include "copy_template.S" : mov x0, #0 0.00 : ffff800010cb7910: mov x0, #0x0 // #0 : ret 0.00 : ffff800010cb7914: ret Percent | Source code & Disassembly of vmlinux for cycles (7 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101f4a60 : : vmstat_shepherd(): : static void vmstat_shepherd(struct work_struct *w); : : static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); : : static void vmstat_shepherd(struct work_struct *w) : { 0.00 : ffff8000101f4a60: stp x29, x30, [sp, #-64]! 0.00 : ffff8000101f4a64: mov x29, sp 0.00 : ffff8000101f4a68: stp x19, x20, [sp, #16] : int cpu; : : get_online_cpus(); : /* Check processors whose vmstat worker threads have been disabled */ : for_each_online_cpu(cpu) { : struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 0.00 : ffff8000101f4a6c: adrp x20, ffff800011526000 0.00 : ffff8000101f4a70: add x20, x20, #0x9b8 : { 0.00 : ffff8000101f4a74: stp x21, x22, [sp, #32] : struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 0.00 : ffff8000101f4a78: add x20, x20, #0x2a0 0.00 : ffff8000101f4a7c: adrp x21, ffff800011909000 0.00 : ffff8000101f4a80: add x21, x21, #0x928 : { 0.00 : ffff8000101f4a84: stp x23, x24, [sp, #48] : for_each_online_cpu(cpu) { 0.00 : ffff8000101f4a88: mov w19, #0xffffffff // #-1 0.00 : ffff8000101f4a8c: adrp x23, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101f4a90: adrp x22, ffff80001190a000 <__per_cpu_offset+0x6d8> : get_online_cpus(): : #endif /* !CONFIG_HOTPLUG_CPU */ : : /* Wrappers which go away once all code is converted */ : static inline void cpu_hotplug_begin(void) { cpus_write_lock(); } : static inline void cpu_hotplug_done(void) { cpus_write_unlock(); } : static inline void get_online_cpus(void) { cpus_read_lock(); } 0.00 : ffff8000101f4a94: bl ffff8000100e7bd8 : vmstat_shepherd(): 41.56 : ffff8000101f4a98: add x1, x23, #0x160 0.00 : ffff8000101f4a9c: mov w0, w19 0.00 : ffff8000101f4aa0: bl ffff800010cb92d8 0.00 : ffff8000101f4aa4: mov w19, w0 0.00 : ffff8000101f4aa8: ldr w2, [x22, #752] : struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 0.00 : ffff8000101f4aac: mov x1, x20 : for_each_online_cpu(cpu) { 0.00 : ffff8000101f4ab0: cmp w0, w2 0.00 : ffff8000101f4ab4: b.cs ffff8000101f4b0c // b.hs, b.nlast : struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 0.00 : ffff8000101f4ab8: ldr x3, [x21, w19, sxtw #3] 0.00 : ffff8000101f4abc: add x24, x1, x3 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 14.65 : ffff8000101f4ac0: ldr x1, [x1, x3] : vmstat_shepherd(): : : if (!delayed_work_pending(dw) && need_update(cpu)) 0.00 : ffff8000101f4ac4: tbnz w1, #0, ffff8000101f4a98 43.79 : ffff8000101f4ac8: bl ffff8000101f3ff0 0.00 : ffff8000101f4acc: tst w0, #0xff 0.00 : ffff8000101f4ad0: b.eq ffff8000101f4a98 // b.none : queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 0.00 : ffff8000101f4ad4: adrp x1, ffff800011b26000 0.00 : ffff8000101f4ad8: mov x2, x24 0.00 : ffff8000101f4adc: mov x3, #0x0 // #0 0.00 : ffff8000101f4ae0: mov w0, w19 0.00 : ffff8000101f4ae4: ldr x1, [x1, #2728] 0.00 : ffff8000101f4ae8: bl ffff800010105988 : for_each_online_cpu(cpu) { 0.00 : ffff8000101f4aec: add x1, x23, #0x160 0.00 : ffff8000101f4af0: mov w0, w19 0.00 : ffff8000101f4af4: bl ffff800010cb92d8 0.00 : ffff8000101f4af8: mov w19, w0 0.00 : ffff8000101f4afc: ldr w2, [x22, #752] : struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 0.00 : ffff8000101f4b00: mov x1, x20 : for_each_online_cpu(cpu) { 0.00 : ffff8000101f4b04: cmp w0, w2 0.00 : ffff8000101f4b08: b.cc ffff8000101f4ab8 // b.lo, b.ul, b.last : put_online_cpus(): : static inline void put_online_cpus(void) { cpus_read_unlock(); } 0.00 : ffff8000101f4b0c: bl ffff8000100e7cf0 : vmstat_shepherd(): : } : put_online_cpus(); : : schedule_delayed_work(&shepherd, 0.00 : ffff8000101f4b10: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101f4b14: ldrsw x0, [x0, #972] 0.00 : ffff8000101f4b18: bl ffff8000101696e0 : queue_delayed_work(): : */ : static inline bool queue_delayed_work(struct workqueue_struct *wq, : struct delayed_work *dwork, : unsigned long delay) : { : return queue_delayed_work_on(WORK_CPU_UNBOUND, wq, dwork, delay); 0.00 : ffff8000101f4b1c: mov x3, x0 : schedule_delayed_work(): : * workqueue. : */ : static inline bool schedule_delayed_work(struct delayed_work *dwork, : unsigned long delay) : { : return queue_delayed_work(system_wq, dwork, delay); 0.00 : ffff8000101f4b20: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : queue_delayed_work(): : return queue_delayed_work_on(WORK_CPU_UNBOUND, wq, dwork, delay); 0.00 : ffff8000101f4b24: adrp x2, ffff80001192b000 0.00 : ffff8000101f4b28: mov w0, #0x100 // #256 0.00 : ffff8000101f4b2c: add x2, x2, #0xb20 0.00 : ffff8000101f4b30: ldr x1, [x1, #496] 0.00 : ffff8000101f4b34: bl ffff800010105988 : vmstat_shepherd(): : round_jiffies_relative(sysctl_stat_interval)); : } 0.00 : ffff8000101f4b38: ldp x19, x20, [sp, #16] 0.00 : ffff8000101f4b3c: ldp x21, x22, [sp, #32] 0.00 : ffff8000101f4b40: ldp x23, x24, [sp, #48] 0.00 : ffff8000101f4b44: ldp x29, x30, [sp], #64 0.00 : ffff8000101f4b48: ret Percent | Source code & Disassembly of vmlinux for cycles (20 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010456f78 : : bio_alloc_bioset(): : * RETURNS: : * Pointer to new bio on success, NULL on failure. : */ : struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, : struct bio_set *bs) : { 0.00 : ffff800010456f78: stp x29, x30, [sp, #-112]! 0.00 : ffff800010456f7c: mov x29, sp 0.00 : ffff800010456f80: stp x19, x20, [sp, #16] 0.00 : ffff800010456f84: mov x20, x2 15.05 : ffff800010456f88: stp x21, x22, [sp, #32] 0.00 : ffff800010456f8c: adrp x21, ffff800011909000 0.00 : ffff800010456f90: str x23, [sp, #48] 0.00 : ffff800010456f94: add x3, x21, #0x908 0.00 : ffff800010456f98: mov w22, w1 0.00 : ffff800010456f9c: mov w23, w0 0.00 : ffff800010456fa0: ldr x4, [x3] 0.00 : ffff800010456fa4: str x4, [x29, #104] 0.00 : ffff800010456fa8: mov x4, #0x0 // #0 : unsigned inline_vecs; : struct bio_vec *bvl = NULL; : struct bio *bio; : void *p; : : if (!bs) { 0.00 : ffff800010456fac: cbz x2, ffff8000104570d0 : gfp_mask); : front_pad = 0; : inline_vecs = nr_iovecs; : } else { : /* should not use nobvec bioset for nr_iovecs > 0 */ : if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && 14.77 : ffff800010456fb0: ldr x0, [x2, #104] 0.00 : ffff800010456fb4: cmp x0, #0x0 0.00 : ffff800010456fb8: ccmp w1, #0x0, #0x4, eq // eq = none 0.00 : ffff800010456fbc: b.ne ffff800010457174 // b.any 4.99 : ffff800010456fc0: stp x24, x25, [x29, #56] 5.03 : ffff800010456fc4: str x26, [x29, #72] 0.00 : ffff800010456fc8: add x26, x2, #0x10 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010456fcc: mrs x0, sp_el0 : bio_alloc_bioset(): : * without __GFP_DIRECT_RECLAIM; if that fails, we punt those : * bios we would be blocking to the rescuer workqueue before : * we retry with the original gfp_flags. : */ : : if (current->bio_list && 5.00 : ffff800010456fd0: ldr x0, [x0, #1832] 0.00 : ffff800010456fd4: cbz x0, ffff8000104570b8 0.00 : ffff800010456fd8: ldr x1, [x0] 0.00 : ffff800010456fdc: cbz x1, ffff8000104570ac : (!bio_list_empty(¤t->bio_list[0]) || : !bio_list_empty(¤t->bio_list[1])) && 0.00 : ffff800010456fe0: ldr x0, [x20, #360] 0.00 : ffff800010456fe4: cbz x0, ffff8000104570b8 : bs->rescue_workqueue) : gfp_mask &= ~__GFP_DIRECT_RECLAIM; 0.00 : ffff800010456fe8: and w25, w23, #0xfffffbff : : p = mempool_alloc(&bs->bio_pool, gfp_mask); 0.00 : ffff800010456fec: mov x0, x26 0.00 : ffff800010456ff0: mov w1, w25 0.00 : ffff800010456ff4: bl ffff8000101d7640 : if (!p && gfp_mask != saved_gfp) { 0.00 : ffff800010456ff8: cmp x0, #0x0 : p = mempool_alloc(&bs->bio_pool, gfp_mask); 0.00 : ffff800010456ffc: mov x24, x0 : if (!p && gfp_mask != saved_gfp) { 0.00 : ffff800010457000: ccmp w23, w25, #0x4, eq // eq = none 0.00 : ffff800010457004: b.eq ffff800010457024 // b.none : punt_bios_to_rescuer(bs); 0.00 : ffff800010457008: mov x0, x20 0.00 : ffff80001045700c: bl ffff800010454f90 : gfp_mask = saved_gfp; : p = mempool_alloc(&bs->bio_pool, gfp_mask); 0.00 : ffff800010457010: mov w1, w23 0.00 : ffff800010457014: mov x0, x26 0.00 : ffff800010457018: mov w25, w23 0.00 : ffff80001045701c: bl ffff8000101d7640 0.00 : ffff800010457020: mov x24, x0 : } : : front_pad = bs->front_pad; 10.12 : ffff800010457024: ldr w19, [x20, #8] : inline_vecs = BIO_INLINE_VECS; : } : : if (unlikely(!p)) 0.00 : ffff800010457028: cbz x24, ffff8000104571a0 : return NULL; : : bio = p + front_pad; 0.00 : ffff80001045702c: add x19, x24, w19, uxtw : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010457030: mov w0, #0x1 // #1 : bio_alloc_bioset(): : bio_init(bio, NULL, 0); : : if (nr_iovecs > inline_vecs) { 0.00 : ffff800010457034: cmp w22, #0x4 : bio_init(): : memset(bio, 0, sizeof(*bio)); 0.00 : ffff800010457038: stp xzr, xzr, [x19, #16] : __write_once_size(): 9.99 : ffff80001045703c: str w0, [x19, #28] : bio_init(): 15.15 : ffff800010457040: stp xzr, xzr, [x19, #96] 5.14 : ffff800010457044: stp xzr, xzr, [x19] 0.00 : ffff800010457048: stp xzr, xzr, [x19, #32] 0.00 : ffff80001045704c: stp xzr, xzr, [x19, #48] 0.00 : ffff800010457050: stp xzr, xzr, [x19, #64] 0.00 : ffff800010457054: stp xzr, xzr, [x19, #80] : __write_once_size(): 5.03 : ffff800010457058: str w0, [x19, #100] : bio_init(): 0.00 : ffff80001045705c: str xzr, [x19, #112] : bio_alloc_bioset(): : if (nr_iovecs > inline_vecs) { 0.00 : ffff800010457060: b.hi ffff800010457120 // b.pmore 9.72 : ffff800010457064: ldp x24, x25, [x29, #56] 0.00 : ffff800010457068: ldr x26, [x29, #72] : if (unlikely(!bvl)) : goto err_free; : : bio->bi_flags |= idx << BVEC_POOL_OFFSET; : } else if (nr_iovecs) { : bvl = bio->bi_inline_vecs; 0.00 : ffff80001045706c: add x0, x19, #0x78 0.00 : ffff800010457070: cmp w22, #0x0 0.00 : ffff800010457074: csel x0, x0, xzr, ne // ne = any : } : : bio->bi_pool = bs; : bio->bi_max_vecs = nr_iovecs; 0.00 : ffff800010457078: strh w22, [x19, #98] : bio->bi_pool = bs; 0.00 : ffff80001045707c: stp x0, x20, [x19, #104] : return bio; : : err_free: : mempool_free(p, &bs->bio_pool); : return NULL; : } 0.00 : ffff800010457080: add x21, x21, #0x908 0.00 : ffff800010457084: mov x0, x19 0.00 : ffff800010457088: ldr x2, [x29, #104] 0.00 : ffff80001045708c: ldr x1, [x21] 0.00 : ffff800010457090: eor x1, x2, x1 0.00 : ffff800010457094: cbnz x1, ffff8000104571d0 0.00 : ffff800010457098: ldp x19, x20, [sp, #16] 0.00 : ffff80001045709c: ldp x21, x22, [sp, #32] 0.00 : ffff8000104570a0: ldr x23, [sp, #48] 0.00 : ffff8000104570a4: ldp x29, x30, [sp], #112 0.00 : ffff8000104570a8: ret : (!bio_list_empty(¤t->bio_list[0]) || 0.00 : ffff8000104570ac: ldr x0, [x0, #16] 0.00 : ffff8000104570b0: cbnz x0, ffff800010456fe0 0.00 : ffff8000104570b4: nop : p = mempool_alloc(&bs->bio_pool, gfp_mask); 0.00 : ffff8000104570b8: mov w1, w23 0.00 : ffff8000104570bc: mov x0, x26 0.00 : ffff8000104570c0: mov w25, w23 0.00 : ffff8000104570c4: bl ffff8000101d7640 0.00 : ffff8000104570c8: mov x24, x0 0.00 : ffff8000104570cc: b ffff800010457024 : if (nr_iovecs > UIO_MAXIOV) 0.00 : ffff8000104570d0: cmp w1, #0x400 0.00 : ffff8000104570d4: b.hi ffff800010457178 // b.pmore : p = kmalloc(sizeof(struct bio) + 0.00 : ffff8000104570d8: ubfiz x0, x22, #4, #32 : kmalloc(): : return kmem_cache_alloc_trace( : kmalloc_caches[kmalloc_type(flags)][index], : flags, size); : #endif : } : return __kmalloc(size, flags); 0.00 : ffff8000104570dc: mov w1, w23 0.00 : ffff8000104570e0: add x0, x0, #0x78 0.00 : ffff8000104570e4: bl ffff800010252fd8 <__kmalloc> 0.00 : ffff8000104570e8: mov x19, x0 : bio_alloc_bioset(): : if (unlikely(!p)) 0.00 : ffff8000104570ec: cbz x0, ffff800010457178 : bio_init(): : memset(bio, 0, sizeof(*bio)); 0.00 : ffff8000104570f0: stp xzr, xzr, [x0, #16] : __write_once_size(): 0.00 : ffff8000104570f4: mov w0, #0x1 // #1 0.00 : ffff8000104570f8: str w0, [x19, #28] : bio_init(): 0.00 : ffff8000104570fc: stp xzr, xzr, [x19, #96] 0.00 : ffff800010457100: stp xzr, xzr, [x19] 0.00 : ffff800010457104: stp xzr, xzr, [x19, #32] 0.00 : ffff800010457108: stp xzr, xzr, [x19, #48] 0.00 : ffff80001045710c: stp xzr, xzr, [x19, #64] 0.00 : ffff800010457110: stp xzr, xzr, [x19, #80] : __write_once_size(): 0.00 : ffff800010457114: str w0, [x19, #100] : bio_init(): 0.00 : ffff800010457118: str xzr, [x19, #112] 0.00 : ffff80001045711c: b ffff80001045706c 0.00 : ffff800010457120: stp x27, x28, [x29, #80] : bio_alloc_bioset(): : unsigned long idx = 0; 0.00 : ffff800010457124: add x28, x29, #0x70 : bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool); 0.00 : ffff800010457128: add x27, x20, #0x58 0.00 : ffff80001045712c: mov w1, w22 0.00 : ffff800010457130: mov x3, x27 0.00 : ffff800010457134: mov w0, w25 : unsigned long idx = 0; 0.00 : ffff800010457138: str xzr, [x28, #-16]! : bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool); 0.00 : ffff80001045713c: mov x2, x28 0.00 : ffff800010457140: bl ffff800010456e48 : if (!bvl && gfp_mask != saved_gfp) { 0.00 : ffff800010457144: cmp x0, #0x0 0.00 : ffff800010457148: ccmp w25, w23, #0x4, eq // eq = none 0.00 : ffff80001045714c: b.ne ffff800010457180 // b.any : if (unlikely(!bvl)) 0.00 : ffff800010457150: cbz x0, ffff8000104571b0 : bio->bi_flags |= idx << BVEC_POOL_OFFSET; 0.00 : ffff800010457154: ldrh w2, [x19, #20] 0.00 : ffff800010457158: ldr x1, [x29, #96] 0.00 : ffff80001045715c: orr w1, w2, w1, lsl #13 0.00 : ffff800010457160: strh w1, [x19, #20] 0.00 : ffff800010457164: ldp x24, x25, [x29, #56] 0.00 : ffff800010457168: ldp x26, x27, [x29, #72] 0.00 : ffff80001045716c: ldr x28, [x29, #88] 0.00 : ffff800010457170: b ffff800010457078 : if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && 0.00 : ffff800010457174: brk #0x800 : return NULL; 0.00 : ffff800010457178: mov x19, #0x0 // #0 0.00 : ffff80001045717c: b ffff800010457080 : punt_bios_to_rescuer(bs); 0.00 : ffff800010457180: mov x0, x20 0.00 : ffff800010457184: bl ffff800010454f90 : bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool); 0.00 : ffff800010457188: mov x3, x27 0.00 : ffff80001045718c: mov x2, x28 0.00 : ffff800010457190: mov w1, w22 0.00 : ffff800010457194: mov w0, w23 0.00 : ffff800010457198: bl ffff800010456e48 0.00 : ffff80001045719c: b ffff800010457150 0.00 : ffff8000104571a0: ldp x24, x25, [x29, #56] : return NULL; 0.00 : ffff8000104571a4: mov x19, #0x0 // #0 0.00 : ffff8000104571a8: ldr x26, [x29, #72] 0.00 : ffff8000104571ac: b ffff800010457080 : mempool_free(p, &bs->bio_pool); 0.00 : ffff8000104571b0: mov x1, x26 0.00 : ffff8000104571b4: mov x0, x24 : return NULL; 0.00 : ffff8000104571b8: mov x19, #0x0 // #0 : mempool_free(p, &bs->bio_pool); 0.00 : ffff8000104571bc: bl ffff8000101d7548 : return NULL; 0.00 : ffff8000104571c0: ldp x24, x25, [x29, #56] 0.00 : ffff8000104571c4: ldp x26, x27, [x29, #72] 0.00 : ffff8000104571c8: ldr x28, [x29, #88] 0.00 : ffff8000104571cc: b ffff800010457080 0.00 : ffff8000104571d0: stp x24, x25, [x29, #56] 0.00 : ffff8000104571d4: stp x26, x27, [x29, #72] 0.00 : ffff8000104571d8: str x28, [x29, #88] : } 0.00 : ffff8000104571dc: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (19 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010487e48 : : iov_iter_init(): : : void iov_iter_init(struct iov_iter *i, unsigned int direction, : const struct iovec *iov, unsigned long nr_segs, : size_t count) : { : WARN_ON(direction & ~(READ | WRITE)); 26.41 : ffff800010487e48: tst w1, #0xfffffffe 0.00 : ffff800010487e4c: b.ne ffff800010487e94 // b.any : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010487e50: mrs x5, sp_el0 : iov_iter_init(): : direction &= READ | WRITE; : : /* It will get better. Eventually... */ : if (uaccess_kernel()) { 21.14 : ffff800010487e54: ldr x5, [x5, #8] : direction &= READ | WRITE; 0.00 : ffff800010487e58: and w1, w1, #0x1 : if (uaccess_kernel()) { 0.00 : ffff800010487e5c: cmn x5, #0x1 0.00 : ffff800010487e60: b.eq ffff800010487e7c // b.none : i->type = ITER_KVEC | direction; : i->kvec = (struct kvec *)iov; : } else { : i->type = ITER_IOVEC | direction; 47.71 : ffff800010487e64: orr w1, w1, #0x4 4.73 : ffff800010487e68: str w1, [x0] : i->iov = iov; : } : i->nr_segs = nr_segs; : i->iov_offset = 0; : i->count = count; 0.00 : ffff800010487e6c: stp xzr, x4, [x0, #8] : i->iov = iov; 0.00 : ffff800010487e70: str x2, [x0, #24] : i->nr_segs = nr_segs; 0.00 : ffff800010487e74: str x3, [x0, #32] : } 0.00 : ffff800010487e78: ret : i->type = ITER_KVEC | direction; 0.00 : ffff800010487e7c: orr w1, w1, #0x8 0.00 : ffff800010487e80: str w1, [x0] : i->count = count; 0.00 : ffff800010487e84: stp xzr, x4, [x0, #8] : i->kvec = (struct kvec *)iov; 0.00 : ffff800010487e88: str x2, [x0, #24] : i->nr_segs = nr_segs; 0.00 : ffff800010487e8c: str x3, [x0, #32] : } 0.00 : ffff800010487e90: ret : WARN_ON(direction & ~(READ | WRITE)); 0.00 : ffff800010487e94: brk #0x800 0.00 : ffff800010487e98: b ffff800010487e50 Percent | Source code & Disassembly of vmlinux for cycles (6 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100ec468 <_local_bh_enable>: : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000100ec468: mrs x0, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000100ec46c: ldr w0, [x0, #16] : _local_bh_enable(): : * Special-case - softirqs can safely be enabled by __do_softirq(), : * without processing still-pending softirqs: : */ : void _local_bh_enable(void) : { : WARN_ON_ONCE(in_irq()); 0.00 : ffff8000100ec470: tst w0, #0xf0000 0.00 : ffff8000100ec474: b.ne ffff8000100ec494 <_local_bh_enable+0x2c> // b.any : get_current(): 16.81 : ffff8000100ec478: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff8000100ec47c: ldr w1, [x0, #16] 83.19 : ffff8000100ec480: ldr w1, [x0, #16] 0.00 : ffff8000100ec484: ldr w1, [x0, #16] : __preempt_count_sub(): : } : : static inline void __preempt_count_sub(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc -= val; 0.00 : ffff8000100ec488: sub w1, w1, #0x200 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000100ec48c: str w1, [x0, #16] : _local_bh_enable(): : __local_bh_enable(SOFTIRQ_DISABLE_OFFSET); : } 0.00 : ffff8000100ec490: ret : WARN_ON_ONCE(in_irq()); 0.00 : ffff8000100ec494: brk #0x800 : get_current(): 0.00 : ffff8000100ec498: mrs x0, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000100ec49c: ldr w1, [x0, #16] 0.00 : ffff8000100ec4a0: ldr w1, [x0, #16] 0.00 : ffff8000100ec4a4: ldr w1, [x0, #16] : __preempt_count_sub(): 0.00 : ffff8000100ec4a8: sub w1, w1, #0x200 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000100ec4ac: str w1, [x0, #16] : _local_bh_enable(): : } 0.00 : ffff8000100ec4b0: ret Percent | Source code & Disassembly of vmlinux for cycles (20 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010489c28 : : iov_iter_advance(): : /* ... and discard everything past that point */ : pipe_truncate(i); : } : : void iov_iter_advance(struct iov_iter *i, size_t size) : { 5.07 : ffff800010489c28: stp x29, x30, [sp, #-80]! 0.00 : ffff800010489c2c: mov x29, sp 0.00 : ffff800010489c30: str x20, [sp, #24] 0.00 : ffff800010489c34: mov x20, x0 : iov_iter_is_pipe(): : return iov_iter_type(i) == ITER_BVEC; : } : : static inline bool iov_iter_is_pipe(const struct iov_iter *i) : { : return iov_iter_type(i) == ITER_PIPE; 0.00 : ffff800010489c38: ldr w0, [x0] 0.00 : ffff800010489c3c: ldr x5, [x20, #16] : iov_iter_type(): : return i->type & ~(READ | WRITE); 0.00 : ffff800010489c40: and w2, w0, #0xfffffffe : iov_iter_advance(): : if (unlikely(iov_iter_is_pipe(i))) { 0.00 : ffff800010489c44: cmp w2, #0x20 0.00 : ffff800010489c48: b.eq ffff800010489cd4 // b.none : pipe_advance(i, size); : return; : } : if (unlikely(iov_iter_is_discard(i))) { 0.00 : ffff800010489c4c: cmp w2, #0x40 0.00 : ffff800010489c50: b.eq ffff800010489ddc // b.none : i->count -= size; : return; : } : iterate_and_advance(i, size, v, 0, 0, 0) 0.00 : ffff800010489c54: cbz x5, ffff800010489cc8 5.03 : ffff800010489c58: cmp x1, x5 0.00 : ffff800010489c5c: str x19, [x29, #16] 0.00 : ffff800010489c60: csel x1, x1, x5, ls // ls = plast 0.00 : ffff800010489c64: ldr x19, [x20, #8] 0.00 : ffff800010489c68: tbnz w0, #4, ffff800010489e2c 0.00 : ffff800010489c6c: tbnz w0, #3, ffff800010489f10 9.69 : ffff800010489c70: tbnz w0, #6, ffff800010489f7c 0.00 : ffff800010489c74: ldr x2, [x20, #24] 0.00 : ffff800010489c78: ldr x4, [x2, #8] 0.00 : ffff800010489c7c: sub x0, x4, x19 0.00 : ffff800010489c80: cmp x0, x1 0.00 : ffff800010489c84: csel x0, x0, x1, ls // ls = plast 5.11 : ffff800010489c88: cbz x0, ffff800010489f84 0.00 : ffff800010489c8c: add x19, x19, x0 0.00 : ffff800010489c90: sub x0, x1, x0 0.00 : ffff800010489c94: mov x3, x2 0.00 : ffff800010489c98: cbnz x0, ffff800010489e0c 15.08 : ffff800010489c9c: cmp x19, x4 0.00 : ffff800010489ca0: b.ne ffff800010489cac // b.any 34.80 : ffff800010489ca4: add x3, x3, #0x10 0.00 : ffff800010489ca8: mov x19, #0x0 // #0 0.00 : ffff800010489cac: ldr x0, [x20, #32] 0.00 : ffff800010489cb0: sub x2, x3, x2 0.00 : ffff800010489cb4: sub x2, x0, x2, asr #4 0.00 : ffff800010489cb8: stp x3, x2, [x20, #24] 0.00 : ffff800010489cbc: sub x1, x5, x1 0.00 : ffff800010489cc0: stp x19, x1, [x20, #8] 25.23 : ffff800010489cc4: ldr x19, [x29, #16] : } 0.00 : ffff800010489cc8: ldr x20, [sp, #24] 0.00 : ffff800010489ccc: ldp x29, x30, [sp], #80 0.00 : ffff800010489cd0: ret 0.00 : ffff800010489cd4: str x19, [x29, #16] 0.00 : ffff800010489cd8: cmp x1, x5 0.00 : ffff800010489cdc: str x21, [x29, #32] 0.00 : ffff800010489ce0: csel x1, x1, x5, ls // ls = plast : pipe_advance(): : struct pipe_inode_info *pipe = i->pipe; 0.00 : ffff800010489ce4: ldr x21, [x20, #24] : if (size) { 0.00 : ffff800010489ce8: cbz x1, ffff800010489d5c : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff800010489cec: ldr w6, [x21, #92] 0.00 : ffff800010489cf0: mov w8, #0x28 // #40 : unsigned int i_head = i->head; 0.00 : ffff800010489cf4: ldr w3, [x20, #32] : if (off) /* make it relative to the beginning of buffer */ 0.00 : ffff800010489cf8: mov x2, x1 : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff800010489cfc: sub w6, w6, #0x1 0.00 : ffff800010489d00: ldr x7, [x21, #144] : size_t off = i->iov_offset, left = size; 0.00 : ffff800010489d04: ldr x4, [x20, #8] 0.00 : ffff800010489d08: and w0, w6, w3 0.00 : ffff800010489d0c: umaddl x0, w0, w8, x7 : if (off) /* make it relative to the beginning of buffer */ 0.00 : ffff800010489d10: cbz x4, ffff800010489d20 : left += off - pipe->bufs[i_head & p_mask].offset; 0.00 : ffff800010489d14: ldr w2, [x0, #8] 0.00 : ffff800010489d18: add x4, x4, x1 0.00 : ffff800010489d1c: sub x2, x4, x2 : if (left <= buf->len) 0.00 : ffff800010489d20: ldr w4, [x0, #12] 0.00 : ffff800010489d24: cmp x4, x2 0.00 : ffff800010489d28: b.cs ffff800010489d4c // b.hs, b.nlast : buf = &pipe->bufs[i_head & p_mask]; 0.00 : ffff800010489d2c: mov w8, #0x28 // #40 : i_head++; 0.00 : ffff800010489d30: add w3, w3, #0x1 : left -= buf->len; 0.00 : ffff800010489d34: sub x2, x2, x4 : buf = &pipe->bufs[i_head & p_mask]; 0.00 : ffff800010489d38: and w0, w6, w3 0.00 : ffff800010489d3c: umaddl x0, w0, w8, x7 : if (left <= buf->len) 0.00 : ffff800010489d40: ldr w4, [x0, #12] 0.00 : ffff800010489d44: cmp x4, x2 0.00 : ffff800010489d48: b.cc ffff800010489d30 // b.lo, b.ul, b.last : i->head = i_head; 0.00 : ffff800010489d4c: str w3, [x20, #32] : i->iov_offset = buf->offset + left; 0.00 : ffff800010489d50: ldr w0, [x0, #8] 0.00 : ffff800010489d54: add x2, x0, x2 0.00 : ffff800010489d58: str x2, [x20, #8] : i->count -= size; 0.00 : ffff800010489d5c: sub x1, x5, x1 0.00 : ffff800010489d60: str x1, [x20, #16] : pipe_truncate(): : if (!pipe_empty(p_head, p_tail)) { 0.00 : ffff800010489d64: ldp w19, w0, [x21, #80] 0.00 : ffff800010489d68: cmp w0, w19 0.00 : ffff800010489d6c: b.eq ffff800010489f8c // b.none 0.00 : ffff800010489d70: stp x22, x23, [x29, #40] : size_t off = i->iov_offset; 0.00 : ffff800010489d74: ldr x0, [x20, #8] : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff800010489d78: ldr w22, [x21, #92] : unsigned int i_head = i->head; 0.00 : ffff800010489d7c: ldr w20, [x20, #32] : unsigned int p_mask = pipe->ring_size - 1; 0.00 : ffff800010489d80: sub w22, w22, #0x1 : if (off) { 0.00 : ffff800010489d84: cbnz x0, ffff800010489de8 : pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]); 0.00 : ffff800010489d88: mov w23, #0x28 // #40 : while (p_head != i_head) { 0.00 : ffff800010489d8c: cmp w19, w20 0.00 : ffff800010489d90: b.eq ffff800010489dc8 // b.none 0.00 : ffff800010489d94: nop : pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]); 0.00 : ffff800010489d98: ldr x1, [x21, #144] : p_head--; 0.00 : ffff800010489d9c: sub w19, w19, #0x1 : pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]); 0.00 : ffff800010489da0: and w2, w22, w19 : pipe_buf_release(): : struct pipe_buffer *buf) : { : const struct pipe_buf_operations *ops = buf->ops; : : buf->ops = NULL; : ops->release(pipe, buf); 0.00 : ffff800010489da4: mov x0, x21 : pipe_truncate(): 0.00 : ffff800010489da8: umaddl x2, w2, w23, x1 : pipe_buf_release(): 0.00 : ffff800010489dac: mov x1, x2 : const struct pipe_buf_operations *ops = buf->ops; 0.00 : ffff800010489db0: ldr x3, [x2, #16] : buf->ops = NULL; 0.00 : ffff800010489db4: str xzr, [x2, #16] : ops->release(pipe, buf); 0.00 : ffff800010489db8: ldr x2, [x3, #8] 0.00 : ffff800010489dbc: blr x2 : pipe_truncate(): : while (p_head != i_head) { 0.00 : ffff800010489dc0: cmp w19, w20 0.00 : ffff800010489dc4: b.ne ffff800010489d98 // b.any : pipe->head = p_head; 0.00 : ffff800010489dc8: str w20, [x21, #80] 0.00 : ffff800010489dcc: ldr x19, [x29, #16] 0.00 : ffff800010489dd0: ldp x21, x22, [x29, #32] 0.00 : ffff800010489dd4: ldr x23, [x29, #48] 0.00 : ffff800010489dd8: b ffff800010489cc8 : iov_iter_advance(): : i->count -= size; 0.00 : ffff800010489ddc: sub x1, x5, x1 0.00 : ffff800010489de0: str x1, [x20, #16] : return; 0.00 : ffff800010489de4: b ffff800010489cc8 : pipe_truncate(): : buf = &pipe->bufs[i_head & p_mask]; 0.00 : ffff800010489de8: ldr x2, [x21, #144] 0.00 : ffff800010489dec: and w1, w22, w20 0.00 : ffff800010489df0: mov w3, #0x28 // #40 0.00 : ffff800010489df4: add w20, w20, #0x1 0.00 : ffff800010489df8: umaddl x1, w1, w3, x2 : buf->len = off - buf->offset; 0.00 : ffff800010489dfc: ldr w2, [x1, #8] 0.00 : ffff800010489e00: sub w0, w0, w2 0.00 : ffff800010489e04: str w0, [x1, #12] 0.00 : ffff800010489e08: b ffff800010489d88 : iov_iter_advance(): : iterate_and_advance(i, size, v, 0, 0, 0) 0.00 : ffff800010489e0c: add x3, x3, #0x10 0.00 : ffff800010489e10: ldr x4, [x3, #8] 0.00 : ffff800010489e14: cmp x0, x4 0.00 : ffff800010489e18: csel x19, x0, x4, ls // ls = plast 0.00 : ffff800010489e1c: cbz x19, ffff800010489e0c 0.00 : ffff800010489e20: subs x0, x0, x19 0.00 : ffff800010489e24: b.eq ffff800010489c9c // b.none 0.00 : ffff800010489e28: b ffff800010489e0c 0.00 : ffff800010489e2c: stp x21, x22, [x29, #32] 0.00 : ffff800010489e30: mov w6, w1 0.00 : ffff800010489e34: mov w2, #0x0 // #0 0.00 : ffff800010489e38: mov x4, #0x0 // #0 0.00 : ffff800010489e3c: ldr x21, [x20, #24] 0.00 : ffff800010489e40: mov w8, #0x1000 // #4096 0.00 : ffff800010489e44: nop 0.00 : ffff800010489e48: lsl x22, x4, #4 0.00 : ffff800010489e4c: add x3, x21, x22 0.00 : ffff800010489e50: cbz w6, ffff800010489ef4 0.00 : ffff800010489e54: ldp w0, w3, [x3, #8] 0.00 : ffff800010489e58: sub w7, w0, w19 0.00 : ffff800010489e5c: add w3, w19, w3 0.00 : ffff800010489e60: cmp w7, w6 0.00 : ffff800010489e64: and w3, w3, #0xfff 0.00 : ffff800010489e68: sub w3, w8, w3 0.00 : ffff800010489e6c: csel w7, w7, w6, ls // ls = plast 0.00 : ffff800010489e70: cmp w3, w7 0.00 : ffff800010489e74: csel w3, w3, w7, ls // ls = plast : bvec_iter_advance(): : static inline bool bvec_iter_advance(const struct bio_vec *bv, : struct bvec_iter *iter, unsigned bytes) : { : unsigned int idx = iter->bi_idx; : : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff800010489e78: cmp w6, w3 0.00 : ffff800010489e7c: b.cc ffff800010489ebc // b.lo, b.ul, b.last : "Attempted to advance past end of bvec iter\n")) { : iter->bi_size = 0; : return false; : } : : iter->bi_size -= bytes; 0.00 : ffff800010489e80: sub w6, w6, w3 : bytes += iter->bi_bvec_done; : : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010489e84: adds w19, w19, w3 0.00 : ffff800010489e88: b.eq ffff800010489e48 // b.none 0.00 : ffff800010489e8c: cmp w0, w19 0.00 : ffff800010489e90: b.hi ffff800010489e48 // b.pmore 0.00 : ffff800010489e94: nop : bytes -= bv[idx].bv_len; : idx++; 0.00 : ffff800010489e98: add w4, w2, #0x1 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010489e9c: subs w19, w19, w0 : idx++; 0.00 : ffff800010489ea0: mov x2, x4 0.00 : ffff800010489ea4: add x0, x21, x4, lsl #4 : while (bytes && bytes >= bv[idx].bv_len) { 0.00 : ffff800010489ea8: b.eq ffff800010489e48 // b.none 0.00 : ffff800010489eac: ldr w0, [x0, #8] 0.00 : ffff800010489eb0: cmp w0, w19 0.00 : ffff800010489eb4: b.ls ffff800010489e98 // b.plast 0.00 : ffff800010489eb8: b ffff800010489e48 : if (WARN_ONCE(bytes > iter->bi_size, 0.00 : ffff800010489ebc: adrp x2, ffff800011ad8000 0.00 : ffff800010489ec0: mov x3, x21 0.00 : ffff800010489ec4: ldrb w0, [x2, #1302] 0.00 : ffff800010489ec8: cbnz w0, ffff800010489ef0 0.00 : ffff800010489ecc: mov w3, #0x1 // #1 0.00 : ffff800010489ed0: str x1, [x29, #72] 0.00 : ffff800010489ed4: strb w3, [x2, #1302] 0.00 : ffff800010489ed8: adrp x0, ffff8000111e9000 0.00 : ffff800010489edc: add x0, x0, #0x680 0.00 : ffff800010489ee0: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010489ee4: brk #0x800 0.00 : ffff800010489ee8: ldp x5, x3, [x20, #16] 0.00 : ffff800010489eec: ldr x1, [x29, #72] 0.00 : ffff800010489ef0: add x3, x3, x22 : iov_iter_advance(): 0.00 : ffff800010489ef4: ldr x2, [x20, #32] 0.00 : ffff800010489ef8: sub x0, x3, x21 0.00 : ffff800010489efc: mov w19, w19 0.00 : ffff800010489f00: sub x0, x2, x0, asr #4 0.00 : ffff800010489f04: stp x3, x0, [x20, #24] 0.00 : ffff800010489f08: ldp x21, x22, [x29, #32] 0.00 : ffff800010489f0c: b ffff800010489cbc 0.00 : ffff800010489f10: ldr x2, [x20, #24] 0.00 : ffff800010489f14: ldr x4, [x2, #8] 0.00 : ffff800010489f18: sub x0, x4, x19 0.00 : ffff800010489f1c: cmp x0, x1 0.00 : ffff800010489f20: csel x0, x0, x1, ls // ls = plast 0.00 : ffff800010489f24: cbz x0, ffff800010489f98 0.00 : ffff800010489f28: add x19, x19, x0 0.00 : ffff800010489f2c: sub x0, x1, x0 0.00 : ffff800010489f30: mov x3, x2 0.00 : ffff800010489f34: cbnz x0, ffff800010489f5c 0.00 : ffff800010489f38: cmp x19, x4 0.00 : ffff800010489f3c: b.ne ffff800010489f48 // b.any 0.00 : ffff800010489f40: add x3, x3, #0x10 0.00 : ffff800010489f44: mov x19, #0x0 // #0 0.00 : ffff800010489f48: ldr x4, [x20, #32] 0.00 : ffff800010489f4c: sub x0, x3, x2 0.00 : ffff800010489f50: sub x0, x4, x0, asr #4 0.00 : ffff800010489f54: stp x3, x0, [x20, #24] 0.00 : ffff800010489f58: b ffff800010489cbc 0.00 : ffff800010489f5c: add x3, x3, #0x10 0.00 : ffff800010489f60: ldr x4, [x3, #8] 0.00 : ffff800010489f64: cmp x0, x4 0.00 : ffff800010489f68: csel x19, x0, x4, ls // ls = plast 0.00 : ffff800010489f6c: cbz x19, ffff800010489f5c 0.00 : ffff800010489f70: subs x0, x0, x19 0.00 : ffff800010489f74: b.eq ffff800010489f38 // b.none 0.00 : ffff800010489f78: b ffff800010489f5c 0.00 : ffff800010489f7c: add x19, x19, x1 0.00 : ffff800010489f80: b ffff800010489cbc 0.00 : ffff800010489f84: mov x0, x1 0.00 : ffff800010489f88: b ffff800010489c94 0.00 : ffff800010489f8c: ldr x19, [x29, #16] 0.00 : ffff800010489f90: ldr x21, [x29, #32] 0.00 : ffff800010489f94: b ffff800010489cc8 0.00 : ffff800010489f98: mov x0, x1 0.00 : ffff800010489f9c: b ffff800010489f30 Percent | Source code & Disassembly of vmlinux for cycles (27 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cb7140 <__arch_copy_from_user>: : __arch_copy_from_user(): : stp \ptr, \regB, [\regC], \val : .endm : : end .req x5 : SYM_FUNC_START(__arch_copy_from_user) : add end, x0, x2 3.73 : ffff800010cb7140: add x5, x0, x2 : C_l .req x11 : C_h .req x12 : D_l .req x13 : D_h .req x14 : : mov dst, dstin 0.00 : ffff800010cb7144: mov x6, x0 : cmp count, #16 0.00 : ffff800010cb7148: cmp x2, #0x10 : /*When memory length is less than 16, the accessed are not aligned.*/ : b.lo .Ltiny15 0.00 : ffff800010cb714c: b.cc ffff800010cb71ec <__arch_copy_from_user+0xac> // b.lo, b.ul, b.last : : neg tmp2, src 3.76 : ffff800010cb7150: neg x4, x1 : ands tmp2, tmp2, #15/* Bytes to reach alignment. */ 0.00 : ffff800010cb7154: ands x4, x4, #0xf : b.eq .LSrcAligned 14.76 : ffff800010cb7158: b.eq ffff800010cb71a0 <__arch_copy_from_user+0x60> // b.none : sub count, count, tmp2 3.29 : ffff800010cb715c: sub x2, x2, x4 : * Copy the leading memory data from src to dst in an increasing : * address order.By this way,the risk of overwriting the source : * memory data is eliminated when the distance between src and : * dst is less than 16. The memory accesses here are alignment. : */ : tbz tmp2, #0, 1f 0.00 : ffff800010cb7160: tbz w4, #0, ffff800010cb7170 <__arch_copy_from_user+0x30> : ldrb1 tmp1w, src, #1 0.00 : ffff800010cb7164: ldrb w3, [x1], #1 0.00 : ffff800010cb7168: nop : strb1 tmp1w, dst, #1 0.00 : ffff800010cb716c: strb w3, [x6], #1 : 1: : tbz tmp2, #1, 2f 0.00 : ffff800010cb7170: tbz w4, #1, ffff800010cb7180 <__arch_copy_from_user+0x40> : ldrh1 tmp1w, src, #2 0.00 : ffff800010cb7174: ldrh w3, [x1], #2 0.00 : ffff800010cb7178: nop : strh1 tmp1w, dst, #2 0.00 : ffff800010cb717c: strh w3, [x6], #2 : 2: : tbz tmp2, #2, 3f 0.00 : ffff800010cb7180: tbz w4, #2, ffff800010cb7190 <__arch_copy_from_user+0x50> : ldr1 tmp1w, src, #4 0.00 : ffff800010cb7184: ldr w3, [x1], #4 0.00 : ffff800010cb7188: nop : str1 tmp1w, dst, #4 0.00 : ffff800010cb718c: str w3, [x6], #4 : 3: : tbz tmp2, #3, .LSrcAligned 0.00 : ffff800010cb7190: tbz w4, #3, ffff800010cb71a0 <__arch_copy_from_user+0x60> : ldr1 tmp1, src, #8 0.00 : ffff800010cb7194: ldr x3, [x1], #8 0.00 : ffff800010cb7198: nop : str1 tmp1, dst, #8 0.00 : ffff800010cb719c: str x3, [x6], #8 : : .LSrcAligned: : cmp count, #64 3.45 : ffff800010cb71a0: cmp x2, #0x40 : b.ge .Lcpy_over64 0.00 : ffff800010cb71a4: b.ge ffff800010cb7230 <__arch_copy_from_user+0xf0> // b.tcont : .Ltail63: : /* : * Copy up to 48 bytes of data. At this point we only need the : * bottom 6 bits of count to be accurate. : */ : ands tmp1, count, #0x30 0.00 : ffff800010cb71a8: ands x3, x2, #0x30 : b.eq .Ltiny15 0.00 : ffff800010cb71ac: b.eq ffff800010cb71ec <__arch_copy_from_user+0xac> // b.none : cmp tmp1w, #0x20 0.00 : ffff800010cb71b0: cmp w3, #0x20 : b.eq 1f 0.00 : ffff800010cb71b4: b.eq ffff800010cb71cc <__arch_copy_from_user+0x8c> // b.none : b.lt 2f 0.00 : ffff800010cb71b8: b.lt ffff800010cb71dc <__arch_copy_from_user+0x9c> // b.tstop : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb71bc: ldp x7, x8, [x1], #16 0.00 : ffff800010cb71c0: nop 0.00 : ffff800010cb71c4: nop : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb71c8: stp x7, x8, [x6], #16 : 1: : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb71cc: ldp x7, x8, [x1], #16 0.00 : ffff800010cb71d0: nop 0.00 : ffff800010cb71d4: nop : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb71d8: stp x7, x8, [x6], #16 : 2: : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb71dc: ldp x7, x8, [x1], #16 0.00 : ffff800010cb71e0: nop 0.00 : ffff800010cb71e4: nop : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb71e8: stp x7, x8, [x6], #16 : * precondition that src address is at least 16 bytes bigger than dst : * address,otherwise some source data will be overwritten when memove : * call memcpy directly. To make memmove simpler and decouple the : * memcpy's dependency on memmove, withdrew the original process. : */ : tbz count, #3, 1f 0.00 : ffff800010cb71ec: tbz w2, #3, ffff800010cb71fc <__arch_copy_from_user+0xbc> : ldr1 tmp1, src, #8 0.00 : ffff800010cb71f0: ldr x3, [x1], #8 0.00 : ffff800010cb71f4: nop : str1 tmp1, dst, #8 0.00 : ffff800010cb71f8: str x3, [x6], #8 : 1: : tbz count, #2, 2f 0.00 : ffff800010cb71fc: tbz w2, #2, ffff800010cb720c <__arch_copy_from_user+0xcc> : ldr1 tmp1w, src, #4 0.00 : ffff800010cb7200: ldr w3, [x1], #4 0.00 : ffff800010cb7204: nop : str1 tmp1w, dst, #4 0.00 : ffff800010cb7208: str w3, [x6], #4 : 2: : tbz count, #1, 3f 0.00 : ffff800010cb720c: tbz w2, #1, ffff800010cb721c <__arch_copy_from_user+0xdc> : ldrh1 tmp1w, src, #2 0.00 : ffff800010cb7210: ldrh w3, [x1], #2 0.00 : ffff800010cb7214: nop : strh1 tmp1w, dst, #2 0.00 : ffff800010cb7218: strh w3, [x6], #2 : 3: : tbz count, #0, .Lexitfunc 0.00 : ffff800010cb721c: tbz w2, #0, ffff800010cb7350 <__arch_copy_from_user+0x210> : ldrb1 tmp1w, src, #1 0.00 : ffff800010cb7220: ldrb w3, [x1], #1 0.00 : ffff800010cb7224: nop : strb1 tmp1w, dst, #1 0.00 : ffff800010cb7228: strb w3, [x6], #1 : : b .Lexitfunc 0.00 : ffff800010cb722c: b ffff800010cb7350 <__arch_copy_from_user+0x210> : : .Lcpy_over64: : subs count, count, #128 0.00 : ffff800010cb7230: subs x2, x2, #0x80 : b.ge .Lcpy_body_large 0.00 : ffff800010cb7234: b.ge ffff800010cb72c0 <__arch_copy_from_user+0x180> // b.tcont : /* : * Less than 128 bytes to copy, so handle 64 here and then jump : * to the tail. : */ : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb7238: ldp x7, x8, [x1], #16 18.38 : ffff800010cb723c: nop 0.00 : ffff800010cb7240: nop : stp1 A_l, A_h, dst, #16 14.95 : ffff800010cb7244: stp x7, x8, [x6], #16 : ldp1 B_l, B_h, src, #16 0.00 : ffff800010cb7248: ldp x9, x10, [x1], #16 3.86 : ffff800010cb724c: nop 0.00 : ffff800010cb7250: nop : ldp1 C_l, C_h, src, #16 0.00 : ffff800010cb7254: ldp x11, x12, [x1], #16 3.78 : ffff800010cb7258: nop 0.00 : ffff800010cb725c: nop : stp1 B_l, B_h, dst, #16 0.00 : ffff800010cb7260: stp x9, x10, [x6], #16 : stp1 C_l, C_h, dst, #16 0.00 : ffff800010cb7264: stp x11, x12, [x6], #16 : ldp1 D_l, D_h, src, #16 15.17 : ffff800010cb7268: ldp x13, x14, [x1], #16 7.37 : ffff800010cb726c: nop 0.00 : ffff800010cb7270: nop : stp1 D_l, D_h, dst, #16 3.72 : ffff800010cb7274: stp x13, x14, [x6], #16 : : tst count, #0x3f 0.00 : ffff800010cb7278: tst x2, #0x3f : b.ne .Ltail63 0.00 : ffff800010cb727c: b.ne ffff800010cb71a8 <__arch_copy_from_user+0x68> // b.any : b .Lexitfunc 0.00 : ffff800010cb7280: b ffff800010cb7350 <__arch_copy_from_user+0x210> 0.00 : ffff800010cb7284: nop 0.00 : ffff800010cb7288: nop 0.00 : ffff800010cb728c: nop 0.00 : ffff800010cb7290: nop 0.00 : ffff800010cb7294: nop 0.00 : ffff800010cb7298: nop 0.00 : ffff800010cb729c: nop 0.00 : ffff800010cb72a0: nop 0.00 : ffff800010cb72a4: nop 0.00 : ffff800010cb72a8: nop 0.00 : ffff800010cb72ac: nop 0.00 : ffff800010cb72b0: nop 0.00 : ffff800010cb72b4: nop 0.00 : ffff800010cb72b8: nop 0.00 : ffff800010cb72bc: nop : * 64 bytes per line this ensures the entire loop is in one line. : */ : .p2align L1_CACHE_SHIFT : .Lcpy_body_large: : /* pre-get 64 bytes data. */ : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb72c0: ldp x7, x8, [x1], #16 0.00 : ffff800010cb72c4: nop 0.00 : ffff800010cb72c8: nop : ldp1 B_l, B_h, src, #16 0.00 : ffff800010cb72cc: ldp x9, x10, [x1], #16 0.00 : ffff800010cb72d0: nop 0.00 : ffff800010cb72d4: nop : ldp1 C_l, C_h, src, #16 0.00 : ffff800010cb72d8: ldp x11, x12, [x1], #16 0.00 : ffff800010cb72dc: nop 0.00 : ffff800010cb72e0: nop : ldp1 D_l, D_h, src, #16 0.00 : ffff800010cb72e4: ldp x13, x14, [x1], #16 0.00 : ffff800010cb72e8: nop 0.00 : ffff800010cb72ec: nop : 1: : /* : * interlace the load of next 64 bytes data block with store of the last : * loaded 64 bytes data. : */ : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb72f0: stp x7, x8, [x6], #16 : ldp1 A_l, A_h, src, #16 0.00 : ffff800010cb72f4: ldp x7, x8, [x1], #16 0.00 : ffff800010cb72f8: nop 0.00 : ffff800010cb72fc: nop : stp1 B_l, B_h, dst, #16 0.00 : ffff800010cb7300: stp x9, x10, [x6], #16 : ldp1 B_l, B_h, src, #16 0.00 : ffff800010cb7304: ldp x9, x10, [x1], #16 0.00 : ffff800010cb7308: nop 0.00 : ffff800010cb730c: nop : stp1 C_l, C_h, dst, #16 0.00 : ffff800010cb7310: stp x11, x12, [x6], #16 : ldp1 C_l, C_h, src, #16 0.00 : ffff800010cb7314: ldp x11, x12, [x1], #16 0.00 : ffff800010cb7318: nop 0.00 : ffff800010cb731c: nop : stp1 D_l, D_h, dst, #16 0.00 : ffff800010cb7320: stp x13, x14, [x6], #16 : ldp1 D_l, D_h, src, #16 0.00 : ffff800010cb7324: ldp x13, x14, [x1], #16 0.00 : ffff800010cb7328: nop 0.00 : ffff800010cb732c: nop : subs count, count, #64 0.00 : ffff800010cb7330: subs x2, x2, #0x40 : b.ge 1b 0.00 : ffff800010cb7334: b.ge ffff800010cb72f0 <__arch_copy_from_user+0x1b0> // b.tcont : stp1 A_l, A_h, dst, #16 0.00 : ffff800010cb7338: stp x7, x8, [x6], #16 : stp1 B_l, B_h, dst, #16 0.00 : ffff800010cb733c: stp x9, x10, [x6], #16 : stp1 C_l, C_h, dst, #16 0.00 : ffff800010cb7340: stp x11, x12, [x6], #16 : stp1 D_l, D_h, dst, #16 0.00 : ffff800010cb7344: stp x13, x14, [x6], #16 : : tst count, #0x3f 0.00 : ffff800010cb7348: tst x2, #0x3f : b.ne .Ltail63 0.00 : ffff800010cb734c: b.ne ffff800010cb71a8 <__arch_copy_from_user+0x68> // b.any : #include "copy_template.S" : mov x0, #0 // Nothing to copy 3.78 : ffff800010cb7350: mov x0, #0x0 // #0 : ret 0.00 : ffff800010cb7354: ret Percent | Source code & Disassembly of vmlinux for cycles (18 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d9490 : : read_events(): : } : : static long read_events(struct kioctx *ctx, long min_nr, long nr, : struct io_event __user *event, : ktime_t until) : { 0.00 : ffff8000102d9490: stp x29, x30, [sp, #-208]! 0.00 : ffff8000102d9494: mov x29, sp 0.00 : ffff8000102d9498: str x19, [sp, #16] 0.00 : ffff8000102d949c: adrp x19, ffff800011909000 0.00 : ffff8000102d94a0: add x5, x19, #0x908 0.00 : ffff8000102d94a4: ldr x6, [x5] 0.00 : ffff8000102d94a8: str x6, [x29, #200] 0.00 : ffff8000102d94ac: mov x6, #0x0 // #0 : long ret = 0; 0.00 : ffff8000102d94b0: str xzr, [x29, #80] : * TASK_RUNNING and return 0 too much - that causes us to spin. That : * will only happen if the mutex_lock() call blocks, and we then find : * the ringbuffer empty. So in practice we should be ok, but it's : * something to be aware of when touching this code. : */ : if (until == 0) 0.00 : ffff8000102d94b4: cbz x4, ffff8000102d9518 11.04 : ffff8000102d94b8: stp x20, x21, [x29, #24] 0.00 : ffff8000102d94bc: mov x21, x0 5.54 : ffff8000102d94c0: stp x22, x23, [x29, #40] 0.00 : ffff8000102d94c4: mov x20, x4 5.66 : ffff8000102d94c8: str x24, [x29, #56] 0.00 : ffff8000102d94cc: mov x22, x1 0.00 : ffff8000102d94d0: mov x23, x2 : aio_read_events(ctx, min_nr, nr, event, &ret); : else : wait_event_interruptible_hrtimeout(ctx->wait, 0.00 : ffff8000102d94d4: add x4, x29, #0x50 33.14 : ffff8000102d94d8: mov x24, x3 0.00 : ffff8000102d94dc: bl ffff8000102d90f8 16.82 : ffff8000102d94e0: tst w0, #0xff 0.00 : ffff8000102d94e4: b.eq ffff8000102d9524 // b.none 0.00 : ffff8000102d94e8: ldp x20, x21, [x29, #24] 11.21 : ffff8000102d94ec: ldp x22, x23, [x29, #40] 0.00 : ffff8000102d94f0: ldr x24, [x29, #56] : aio_read_events(ctx, min_nr, nr, event, &ret), : until); : return ret; : } 0.00 : ffff8000102d94f4: add x19, x19, #0x908 5.52 : ffff8000102d94f8: ldr x0, [x29, #80] 0.00 : ffff8000102d94fc: ldr x2, [x29, #200] 5.48 : ffff8000102d9500: ldr x1, [x19] 0.00 : ffff8000102d9504: eor x1, x2, x1 0.00 : ffff8000102d9508: cbnz x1, ffff8000102d95d8 0.00 : ffff8000102d950c: ldr x19, [sp, #16] 5.59 : ffff8000102d9510: ldp x29, x30, [sp], #208 0.00 : ffff8000102d9514: ret : aio_read_events(ctx, min_nr, nr, event, &ret); 0.00 : ffff8000102d9518: add x4, x29, #0x50 0.00 : ffff8000102d951c: bl ffff8000102d90f8 0.00 : ffff8000102d9520: b ffff8000102d94f4 : hrtimer_init_sleeper_on_stack(): : : static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl, : clockid_t clock_id, : enum hrtimer_mode mode) : { : hrtimer_init_sleeper(sl, clock_id, mode); 0.00 : ffff8000102d9524: mov w2, #0x1 // #1 0.00 : ffff8000102d9528: add x0, x29, #0x80 0.00 : ffff8000102d952c: mov w1, w2 0.00 : ffff8000102d9530: str x25, [x29, #64] 0.00 : ffff8000102d9534: bl ffff80001016b580 : read_events(): : wait_event_interruptible_hrtimeout(ctx->wait, 0.00 : ffff8000102d9538: mov x0, #0x7fffffffffffffff // #9223372036854775807 0.00 : ffff8000102d953c: cmp x20, x0 0.00 : ffff8000102d9540: b.eq ffff8000102d955c // b.none : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000102d9544: mrs x0, sp_el0 : read_events(): 0.00 : ffff8000102d9548: ldr x2, [x0, #2392] 0.00 : ffff8000102d954c: mov w3, #0x1 // #1 0.00 : ffff8000102d9550: mov x1, x20 0.00 : ffff8000102d9554: add x0, x29, #0x80 0.00 : ffff8000102d9558: bl ffff80001016b750 0.00 : ffff8000102d955c: add x25, x21, #0x1a0 0.00 : ffff8000102d9560: mov w1, #0x0 // #0 0.00 : ffff8000102d9564: add x0, x29, #0x58 0.00 : ffff8000102d9568: bl ffff80001012fbd8 0.00 : ffff8000102d956c: b ffff8000102d9580 0.00 : ffff8000102d9570: cbnz x20, ffff8000102d95c0 0.00 : ffff8000102d9574: ldr x0, [x29, #192] 0.00 : ffff8000102d9578: cbz x0, ffff8000102d95b4 0.00 : ffff8000102d957c: bl ffff800010cd2a78 0.00 : ffff8000102d9580: mov w2, #0x1 // #1 0.00 : ffff8000102d9584: add x1, x29, #0x58 0.00 : ffff8000102d9588: mov x0, x25 0.00 : ffff8000102d958c: bl ffff80001012fcd0 0.00 : ffff8000102d9590: add x4, x29, #0x50 0.00 : ffff8000102d9594: mov x20, x0 0.00 : ffff8000102d9598: mov x3, x24 0.00 : ffff8000102d959c: mov x2, x23 0.00 : ffff8000102d95a0: mov x1, x22 0.00 : ffff8000102d95a4: mov x0, x21 0.00 : ffff8000102d95a8: bl ffff8000102d90f8 0.00 : ffff8000102d95ac: tst w0, #0xff 0.00 : ffff8000102d95b0: b.eq ffff8000102d9570 // b.none 0.00 : ffff8000102d95b4: add x1, x29, #0x58 0.00 : ffff8000102d95b8: mov x0, x25 0.00 : ffff8000102d95bc: bl ffff80001012fc00 0.00 : ffff8000102d95c0: add x0, x29, #0x80 0.00 : ffff8000102d95c4: bl ffff80001016bad0 0.00 : ffff8000102d95c8: ldp x20, x21, [x29, #24] 0.00 : ffff8000102d95cc: ldp x22, x23, [x29, #40] 0.00 : ffff8000102d95d0: ldp x24, x25, [x29, #56] 0.00 : ffff8000102d95d4: b ffff8000102d94f4 0.00 : ffff8000102d95d8: stp x20, x21, [x29, #24] 0.00 : ffff8000102d95dc: stp x22, x23, [x29, #40] 0.00 : ffff8000102d95e0: stp x24, x25, [x29, #56] : } 0.00 : ffff8000102d95e4: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (19 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101d1bd0 : : filemap_check_errors(): : for (i = 0; i < pagevec_count(pvec); i++) : page_cache_free_page(mapping, pvec->pages[i]); : } : : int filemap_check_errors(struct address_space *mapping) : { 0.00 : ffff8000101d1bd0: stp x29, x30, [sp, #-32]! 0.00 : ffff8000101d1bd4: adrp x1, ffff800011909000 0.00 : ffff8000101d1bd8: add x2, x1, #0x908 0.00 : ffff8000101d1bdc: mov x29, sp 0.00 : ffff8000101d1be0: ldr x3, [x2] 5.24 : ffff8000101d1be4: str x3, [x29, #24] 0.00 : ffff8000101d1be8: mov x3, #0x0 // #0 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 5.23 : ffff8000101d1bec: ldr x2, [x0, #120] : filemap_check_errors(): : int ret = 0; : /* Check for outstanding write errors */ : if (test_bit(AS_ENOSPC, &mapping->flags) && 0.00 : ffff8000101d1bf0: tst w2, #0x2 0.00 : ffff8000101d1bf4: b.ne ffff8000101d1c24 // b.any : int ret = 0; 0.00 : ffff8000101d1bf8: mov w2, #0x0 // #0 : test_bit(): 0.00 : ffff8000101d1bfc: ldr x3, [x0, #120] : filemap_check_errors(): : test_and_clear_bit(AS_ENOSPC, &mapping->flags)) : ret = -ENOSPC; : if (test_bit(AS_EIO, &mapping->flags) && 0.00 : ffff8000101d1c00: tbnz w3, #0, ffff8000101d1c4c : test_and_clear_bit(AS_EIO, &mapping->flags)) : ret = -EIO; : return ret; : } 0.00 : ffff8000101d1c04: add x1, x1, #0x908 0.00 : ffff8000101d1c08: mov w0, w2 42.14 : ffff8000101d1c0c: ldr x2, [x29, #24] 42.11 : ffff8000101d1c10: ldr x1, [x1] 0.00 : ffff8000101d1c14: eor x1, x2, x1 0.00 : ffff8000101d1c18: cbnz x1, ffff8000101d1ca0 5.27 : ffff8000101d1c1c: ldp x29, x30, [sp], #32 0.00 : ffff8000101d1c20: ret : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101d1c24: ldr x2, [x0, #120] : test_and_clear_bit(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (!(READ_ONCE(*p) & mask)) 0.00 : ffff8000101d1c28: tbz w2, #1, ffff8000101d1bf8 : filemap_check_errors(): : test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 0.00 : ffff8000101d1c2c: add x3, x0, #0x78 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101d1c30: b ffff8000101d1c90 0.00 : ffff8000101d1c34: b ffff8000101d1c90 : __lse_atomic64_fetch_andnot(): : ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \ : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) 0.00 : ffff8000101d1c38: mov x2, #0x2 // #2 0.00 : ffff8000101d1c3c: ldclral x2, x2, [x3] : filemap_check_errors(): : if (test_bit(AS_ENOSPC, &mapping->flags) && 0.00 : ffff8000101d1c40: tbz w2, #1, ffff8000101d1bf8 : ret = -ENOSPC; 0.00 : ffff8000101d1c44: mov w2, #0xffffffe4 // #-28 0.00 : ffff8000101d1c48: b ffff8000101d1bfc : __read_once_size(): 0.00 : ffff8000101d1c4c: ldr x3, [x0, #120] : test_and_clear_bit(): 0.00 : ffff8000101d1c50: tbz w3, #0, ffff8000101d1c04 : filemap_check_errors(): : test_and_clear_bit(AS_EIO, &mapping->flags)) 0.00 : ffff8000101d1c54: add x4, x0, #0x78 : arch_static_branch_jump(): 0.00 : ffff8000101d1c58: b ffff8000101d1c7c 0.00 : ffff8000101d1c5c: b ffff8000101d1c7c : __lse_atomic64_fetch_andnot(): 0.00 : ffff8000101d1c60: mov x3, #0x1 // #1 0.00 : ffff8000101d1c64: ldclral x3, x3, [x4] 0.00 : ffff8000101d1c68: mov x0, x3 : filemap_check_errors(): : ret = -EIO; 0.00 : ffff8000101d1c6c: tst x0, #0x1 0.00 : ffff8000101d1c70: mov w0, #0xfffffffb // #-5 0.00 : ffff8000101d1c74: csel w2, w2, w0, eq // eq = none 0.00 : ffff8000101d1c78: b ffff8000101d1c04 : __ll_sc_atomic64_fetch_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff8000101d1c7c: mov x4, #0x1 // #1 0.00 : ffff8000101d1c80: add x0, x0, #0x78 0.00 : ffff8000101d1c84: b ffff8000101d6d20 0.00 : ffff8000101d1c88: mov x0, x3 0.00 : ffff8000101d1c8c: b ffff8000101d1c6c 0.00 : ffff8000101d1c90: mov x3, #0x2 // #2 0.00 : ffff8000101d1c94: add x6, x0, #0x78 0.00 : ffff8000101d1c98: b ffff8000101d6d3c 0.00 : ffff8000101d1c9c: b ffff8000101d1c40 : filemap_check_errors(): : } 0.00 : ffff8000101d1ca0: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (13 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015ff80 : : rcu_preempt_deferred_qs_irqrestore(): : * be quite short, for example, in the case of the call from : * rcu_read_unlock_special(). : */ : static void : rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) : { 7.67 : ffff80001015ff80: stp x29, x30, [sp, #-64]! : * If RCU core is waiting for this CPU to exit its critical section, : * report the fact that it has exited. Because irqs are disabled, : * t->rcu_read_unlock_special cannot change. : */ : special = t->rcu_read_unlock_special; : rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff80001015ff84: adrp x2, ffff800011529000 0.00 : ffff80001015ff88: add x2, x2, #0xac0 : { 0.00 : ffff80001015ff8c: mov x29, sp 0.00 : ffff80001015ff90: str x20, [sp, #24] : special = t->rcu_read_unlock_special; 0.00 : ffff80001015ff94: ldr w20, [x0, #780] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001015ff98: mrs x4, tpidr_el1 : rcu_preempt_deferred_qs_irqrestore(): : rdp = this_cpu_ptr(&rcu_data); 0.00 : ffff80001015ff9c: add x4, x2, x4 : if (!special.s && !rdp->exp_deferred_qs) { 0.00 : ffff80001015ffa0: cbnz w20, ffff80001015ffac 0.00 : ffff80001015ffa4: ldrb w2, [x4, #21] 0.00 : ffff80001015ffa8: cbz w2, ffff8000101600fc 0.00 : ffff80001015ffac: str x19, [x29, #16] : local_irq_restore(flags); : return; : } : t->rcu_read_unlock_special.s = 0; : if (special.b.need_qs) 0.00 : ffff80001015ffb0: tst x20, #0xff00 0.00 : ffff80001015ffb4: str x21, [x29, #32] 0.00 : ffff80001015ffb8: mov x19, x0 : t->rcu_read_unlock_special.s = 0; 0.00 : ffff80001015ffbc: str wzr, [x0, #780] 0.00 : ffff80001015ffc0: mov x21, x1 : if (special.b.need_qs) 0.00 : ffff80001015ffc4: b.ne ffff8000101600c4 // b.any : * Respond to a request by an expedited grace period for a : * quiescent state from this CPU. Note that requests from : * tasks are handled when removing the task from the : * blocked-tasks list below. : */ : if (rdp->exp_deferred_qs) 0.00 : ffff80001015ffc8: ldrb w0, [x4, #21] 0.00 : ffff80001015ffcc: cbnz w0, ffff8000101600d0 : rcu_report_exp_rdp(rdp); : : /* Clean up if blocked during RCU read-side critical section. */ : if (special.b.blocked) { 0.00 : ffff80001015ffd0: and w20, w20, #0xff 0.00 : ffff80001015ffd4: cbz w20, ffff8000101600a8 : * Remove this task from the list it blocked on. The task : * now remains queued on the rcu_node corresponding to the : * CPU it first blocked on, so there is no longer any need : * to loop. Retain a WARN_ON_ONCE() out of sheer paranoia. : */ : rnp = t->rcu_blocked_node; 0.00 : ffff80001015ffd8: ldr x20, [x19, #800] : raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ 0.00 : ffff80001015ffdc: mov x0, x20 0.00 : ffff80001015ffe0: bl ffff800010cd80f0 <_raw_spin_lock> : WARN_ON_ONCE(rnp != t->rcu_blocked_node); 0.00 : ffff80001015ffe4: ldr x0, [x19, #800] 0.00 : ffff80001015ffe8: cmp x0, x20 0.00 : ffff80001015ffec: b.ne ffff8000101601e4 // b.any : WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); 0.00 : ffff80001015fff0: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff80001015fff4: ldrb w1, [x20, #113] 0.00 : ffff80001015fff8: ldr w0, [x0, #676] 0.00 : ffff80001015fffc: sub w0, w0, #0x1 7.64 : ffff800010160000: cmp w1, w0 0.00 : ffff800010160004: b.ne ffff8000101601ec // b.any : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010160008: ldr x5, [x20, #144] : rcu_preempt_deferred_qs_irqrestore(): : empty_norm = !rcu_preempt_blocked_readers_cgp(rnp); : WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq && 7.70 : ffff80001016000c: ldr x0, [x20, #8] 30.70 : ffff800010160010: ldr x1, [x20, #24] 0.00 : ffff800010160014: cmp x1, x0 0.00 : ffff800010160018: b.eq ffff8000101601a0 // b.none : sync_rcu_exp_done(): : * for the current expedited grace period. : */ : static bool sync_rcu_exp_done(struct rcu_node *rnp) : { : raw_lockdep_assert_held_rcu_node(rnp); : return rnp->exp_tasks == NULL && 0.00 : ffff80001016001c: ldr x0, [x20, #152] 0.00 : ffff800010160020: mov w6, #0x1 // #1 0.00 : ffff800010160024: cbz x0, ffff8000101601ac : rcu_preempt_deferred_qs_irqrestore(): : (!empty_norm || rnp->qsmask)); : empty_exp = sync_rcu_exp_done(rnp); : smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ 0.00 : ffff800010160028: dmb ish : rcu_next_node_entry(): : np = t->rcu_node_entry.next; 15.61 : ffff80001016002c: ldr x0, [x19, #784] : rcu_preempt_deferred_qs_irqrestore(): : np = rcu_next_node_entry(t, rnp); : list_del_init(&t->rcu_node_entry); 0.00 : ffff800010160030: add x2, x19, #0x310 : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff800010160034: ldr x4, [x19, #792] : rcu_next_node_entry(): : if (np == &rnp->blkd_tasks) 0.00 : ffff800010160038: add x1, x20, #0x80 0.00 : ffff80001016003c: cmp x0, x1 : __list_del(): : next->prev = prev; 7.36 : ffff800010160040: str x4, [x0, #8] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010160044: str x0, [x4] : rcu_next_node_entry(): 0.00 : ffff800010160048: csel x0, x0, xzr, ne // ne = any : __write_once_size(): 0.00 : ffff80001016004c: str x2, [x19, #784] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff800010160050: str x2, [x19, #792] : rcu_preempt_deferred_qs_irqrestore(): : t->rcu_blocked_node = NULL; 0.00 : ffff800010160054: str xzr, [x19, #800] : trace_rcu_unlock_preempted_task(TPS("rcu_preempt"), : rnp->gp_seq, t->pid); : if (&t->rcu_node_entry == rnp->gp_tasks) 0.00 : ffff800010160058: ldr x1, [x20, #144] 0.00 : ffff80001016005c: cmp x2, x1 0.00 : ffff800010160060: b.eq ffff8000101601c8 // b.none : WRITE_ONCE(rnp->gp_tasks, np); : if (&t->rcu_node_entry == rnp->exp_tasks) 0.00 : ffff800010160064: ldr x1, [x20, #152] 0.00 : ffff800010160068: cmp x2, x1 0.00 : ffff80001016006c: b.eq ffff8000101601bc // b.none : sync_rcu_exp_done(): 0.00 : ffff800010160070: mov w19, #0x0 // #0 0.00 : ffff800010160074: cbz x1, ffff800010160108 : rcu_preempt_deferred_qs_irqrestore(): : * we aren't waiting on any CPUs, report the quiescent state. : * Note that rcu_report_unblock_qs_rnp() releases rnp->lock, : * so we must take a snapshot of the expedited state. : */ : empty_exp_now = sync_rcu_exp_done(rnp); : if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) { 7.72 : ffff800010160078: cbz x5, ffff800010160084 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001016007c: ldr x0, [x20, #144] : rcu_preempt_deferred_qs_irqrestore(): 0.00 : ffff800010160080: cbz x0, ffff800010160118 : rnp->grplo, : rnp->grphi, : !!rnp->gp_tasks); : rcu_report_unblock_qs_rnp(rnp, flags); : } else { : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 7.92 : ffff800010160084: mov x1, x21 0.00 : ffff800010160088: mov x0, x20 0.00 : ffff80001016008c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : : /* : * If this was the last task on the expedited lists, : * then we need to report up the rcu_node hierarchy. : */ : if (!empty_exp && empty_exp_now) 0.00 : ffff800010160090: cbnz w19, ffff800010160174 0.00 : ffff800010160094: ldr x19, [x29, #16] 0.00 : ffff800010160098: ldr x21, [x29, #32] : rcu_report_exp_rnp(rnp, true); : } else { : local_irq_restore(flags); : } : } 0.00 : ffff80001016009c: ldr x20, [sp, #24] 0.00 : ffff8000101600a0: ldp x29, x30, [sp], #64 0.00 : ffff8000101600a4: ret : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff8000101600a8: msr daif, x21 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101600ac: nop 0.00 : ffff8000101600b0: ldr x19, [x29, #16] 0.00 : ffff8000101600b4: ldr x21, [x29, #32] : rcu_preempt_deferred_qs_irqrestore(): 0.00 : ffff8000101600b8: ldr x20, [sp, #24] 0.00 : ffff8000101600bc: ldp x29, x30, [sp], #64 0.00 : ffff8000101600c0: ret : rcu_qs(); 0.00 : ffff8000101600c4: bl ffff80001015df08 : if (rdp->exp_deferred_qs) 0.00 : ffff8000101600c8: ldrb w0, [x4, #21] 0.00 : ffff8000101600cc: cbz w0, ffff80001015ffd0 : rcu_report_exp_rdp(): : * Report expedited quiescent state for specified rcu_data (CPU). : */ : static void rcu_report_exp_rdp(struct rcu_data *rdp) : { : WRITE_ONCE(rdp->exp_deferred_qs, false); : rcu_report_exp_cpu_mult(rdp->mynode, rdp->grpmask, true); 0.00 : ffff8000101600d0: ldp x0, x1, [x4, #24] : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff8000101600d4: strb wzr, [x4, #21] : rcu_report_exp_rdp(): 0.00 : ffff8000101600d8: mov w2, #0x1 // #1 0.00 : ffff8000101600dc: bl ffff80001015c340 0.00 : ffff8000101600e0: b ffff80001015ffd0 : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff8000101600e4: dsb sy 0.00 : ffff8000101600e8: ldr x19, [x29, #16] 0.00 : ffff8000101600ec: ldr x21, [x29, #32] : rcu_preempt_deferred_qs_irqrestore(): : } 0.00 : ffff8000101600f0: ldr x20, [sp, #24] 0.00 : ffff8000101600f4: ldp x29, x30, [sp], #64 0.00 : ffff8000101600f8: ret : arch_local_irq_restore(): : asm volatile(ALTERNATIVE( 0.00 : ffff8000101600fc: msr daif, x1 : arch_static_branch(): 0.00 : ffff800010160100: nop 0.00 : ffff800010160104: b ffff8000101600b8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010160108: ldr x0, [x20, #64] : sync_rcu_exp_done(): : return rnp->exp_tasks == NULL && 0.00 : ffff80001016010c: cmp x0, #0x0 0.00 : ffff800010160110: csel w19, w6, wzr, eq // eq = none 0.00 : ffff800010160114: b ffff800010160078 : __read_once_size(): 0.00 : ffff800010160118: ldr x0, [x20, #144] : rcu_report_unblock_qs_rnp(): : unsigned long mask; : struct rcu_node *rnp_p; : : raw_lockdep_assert_held_rcu_node(rnp); : if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RCU)) || : WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) || 0.00 : ffff80001016011c: cbnz x0, ffff800010160204 0.00 : ffff800010160120: ldr x0, [x20, #32] 0.00 : ffff800010160124: cbnz x0, ffff800010160084 0.00 : ffff800010160128: stp x22, x23, [x29, #40] : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); : return; /* Still need more quiescent states! */ : } : : rnp->completedqs = rnp->gp_seq; : rnp_p = rnp->parent; 0.00 : ffff80001016012c: ldr x22, [x20, #120] : rnp->completedqs = rnp->gp_seq; 0.00 : ffff800010160130: ldr x23, [x20, #8] 0.00 : ffff800010160134: str x23, [x20, #24] : if (rnp_p == NULL) { 0.00 : ffff800010160138: cbz x22, ffff8000101601f4 0.00 : ffff80001016013c: str x24, [x29, #56] : } : : /* Report up the rest of the hierarchy, tracking current ->gp_seq. */ : gps = rnp->gp_seq; : mask = rnp->grpmask; : raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ 0.00 : ffff800010160140: mov x0, x20 : mask = rnp->grpmask; 0.00 : ffff800010160144: ldr x24, [x20, #96] : raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ 0.00 : ffff800010160148: bl ffff800010cd7c38 <_raw_spin_unlock> : raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */ 0.00 : ffff80001016014c: mov x0, x22 0.00 : ffff800010160150: bl ffff800010cd80f0 <_raw_spin_lock> : rcu_report_qs_rnp(mask, rnp_p, gps, flags); 0.00 : ffff800010160154: mov x2, x23 0.00 : ffff800010160158: mov x1, x22 0.00 : ffff80001016015c: mov x0, x24 0.00 : ffff800010160160: mov x3, x21 0.00 : ffff800010160164: bl ffff80001015cda8 0.00 : ffff800010160168: ldr x24, [x29, #56] 0.00 : ffff80001016016c: ldp x22, x23, [x29, #40] : rcu_preempt_deferred_qs_irqrestore(): : if (!empty_exp && empty_exp_now) 0.00 : ffff800010160170: cbz w19, ffff800010160094 : rcu_report_exp_rnp(): : raw_spin_lock_irqsave_rcu_node(rnp, flags); 0.00 : ffff800010160174: mov x0, x20 0.00 : ffff800010160178: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : __rcu_report_exp_rnp(rnp, wake, flags); 0.00 : ffff80001016017c: mov w1, #0x1 // #1 0.00 : ffff800010160180: mov x2, x0 0.00 : ffff800010160184: mov x0, x20 0.00 : ffff800010160188: bl ffff80001015c238 <__rcu_report_exp_rnp> 0.00 : ffff80001016018c: ldr x19, [x29, #16] 0.00 : ffff800010160190: ldr x21, [x29, #32] : rcu_preempt_deferred_qs_irqrestore(): : } 0.00 : ffff800010160194: ldr x20, [sp, #24] 0.00 : ffff800010160198: ldp x29, x30, [sp], #64 0.00 : ffff80001016019c: ret : WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq && 0.00 : ffff8000101601a0: cbz x5, ffff8000101601d8 0.00 : ffff8000101601a4: brk #0x800 0.00 : ffff8000101601a8: b ffff80001016001c : __read_once_size(): 7.67 : ffff8000101601ac: ldr x0, [x20, #64] 0.00 : ffff8000101601b0: cmp x0, #0x0 0.00 : ffff8000101601b4: cset w6, ne // ne = any 0.00 : ffff8000101601b8: b ffff800010160028 : rcu_preempt_deferred_qs_irqrestore(): : rnp->exp_tasks = np; 0.00 : ffff8000101601bc: mov x1, x0 0.00 : ffff8000101601c0: str x0, [x20, #152] 0.00 : ffff8000101601c4: b ffff800010160070 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101601c8: str x0, [x20, #144] 0.00 : ffff8000101601cc: b ffff800010160064 : arch_local_irq_restore(): : pmr_sync(); 0.00 : ffff8000101601d0: dsb sy 0.00 : ffff8000101601d4: b ffff8000101600b8 : rcu_preempt_deferred_qs_irqrestore(): : WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq && 0.00 : ffff8000101601d8: ldr x0, [x20, #32] 0.00 : ffff8000101601dc: cbz x0, ffff80001016001c 0.00 : ffff8000101601e0: b ffff8000101601a4 : WARN_ON_ONCE(rnp != t->rcu_blocked_node); 0.00 : ffff8000101601e4: brk #0x800 0.00 : ffff8000101601e8: b ffff80001015fff0 : WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); 0.00 : ffff8000101601ec: brk #0x800 0.00 : ffff8000101601f0: b ffff800010160008 : rcu_report_unblock_qs_rnp(): : rcu_report_qs_rsp(flags); 0.00 : ffff8000101601f4: mov x0, x21 0.00 : ffff8000101601f8: bl ffff80001015cd58 0.00 : ffff8000101601fc: ldp x22, x23, [x29, #40] 0.00 : ffff800010160200: b ffff800010160090 : WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) || 0.00 : ffff800010160204: brk #0x800 0.00 : ffff800010160208: b ffff800010160084 Percent | Source code & Disassembly of vmlinux for cycles (5 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010136508 : : cpuacct_account_field(): : * Add user/system time to cpuacct. : * : * Note: it's the caller that updates the account of the root cgroup. : */ : void cpuacct_account_field(struct task_struct *tsk, int index, u64 val) : { 19.77 : ffff800010136508: stp x29, x30, [sp, #-48]! 0.00 : ffff80001013650c: mov x29, sp 0.00 : ffff800010136510: str x21, [sp, #32] 0.00 : ffff800010136514: mov x21, x0 0.00 : ffff800010136518: stp x19, x20, [sp, #16] 0.00 : ffff80001013651c: mov w19, w1 0.00 : ffff800010136520: mov x20, x2 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff800010136524: bl ffff80001015c1b0 <__rcu_read_lock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010136528: ldr x0, [x21, #2000] : cpuacct_account_field(): : struct cpuacct *ca; : : rcu_read_lock(); : for (ca = task_ca(tsk); ca != &root_cpuacct; ca = parent_ca(ca)) 0.00 : ffff80001013652c: adrp x5, ffff800011920000 0.00 : ffff800010136530: add x5, x5, #0xa60 : task_css(): : * See task_css_check(). : */ : static inline struct cgroup_subsys_state *task_css(struct task_struct *task, : int subsys_id) : { : return task_css_check(task, subsys_id, false); 0.00 : ffff800010136534: ldr x3, [x0, #16] : cpuacct_account_field(): 0.00 : ffff800010136538: cmp x3, x5 0.00 : ffff80001013653c: b.eq ffff80001013656c // b.none 0.00 : ffff800010136540: sbfiz x1, x19, #3, #32 0.00 : ffff800010136544: nop : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010136548: mrs x4, tpidr_el1 : cpuacct_account_field(): : this_cpu_ptr(ca->cpustat)->cpustat[index] += val; 0.00 : ffff80001013654c: ldr x0, [x3, #248] 0.00 : ffff800010136550: add x0, x0, x4 0.00 : ffff800010136554: ldr x4, [x0, x1] 0.00 : ffff800010136558: add x4, x4, x20 0.00 : ffff80001013655c: str x4, [x0, x1] : for (ca = task_ca(tsk); ca != &root_cpuacct; ca = parent_ca(ca)) 0.00 : ffff800010136560: ldr x3, [x3, #232] 0.00 : ffff800010136564: cmp x3, x5 0.00 : ffff800010136568: b.ne ffff800010136548 // b.any : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 40.19 : ffff80001013656c: bl ffff800010160d20 <__rcu_read_unlock> : cpuacct_account_field(): : rcu_read_unlock(); : } 0.00 : ffff800010136570: ldr x21, [sp, #32] 40.04 : ffff800010136574: ldp x19, x20, [sp, #16] 0.00 : ffff800010136578: ldp x29, x30, [sp], #48 0.00 : ffff80001013657c: ret Percent | Source code & Disassembly of vmlinux for cycles (5 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101b4258 : : acct_account_cputime(): : /** : * acct_account_cputime - update mm integral after cputime update : * @tsk: task_struct for accounting : */ : void acct_account_cputime(struct task_struct *tsk) : { 59.49 : ffff8000101b4258: stp x29, x30, [sp, #-16]! 0.00 : ffff8000101b425c: mov x29, sp : __acct_update_integrals(tsk, tsk->utime, tsk->stime); 20.57 : ffff8000101b4260: ldr x1, [x0, #1360] 0.00 : ffff8000101b4264: ldr x2, [x0, #1368] 0.00 : ffff8000101b4268: bl ffff8000101b3e98 <__acct_update_integrals> : } 19.94 : ffff8000101b426c: ldp x29, x30, [sp], #16 0.00 : ffff8000101b4270: ret Percent | Source code & Disassembly of vmlinux for cycles (5 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010117f78 : : nr_iowait_cpu(): : * it does become runnable. : */ : : unsigned long nr_iowait_cpu(int cpu) : { : return atomic_read(&cpu_rq(cpu)->nr_iowait); 18.96 : ffff800010117f78: adrp x2, ffff800011909000 0.00 : ffff800010117f7c: add x2, x2, #0x928 0.00 : ffff800010117f80: adrp x1, ffff800011528000 0.00 : ffff800010117f84: add x1, x1, #0xe80 60.33 : ffff800010117f88: ldr x0, [x2, w0, sxtw #3] 0.00 : ffff800010117f8c: add x1, x1, x0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 20.70 : ffff800010117f90: ldr w0, [x1, #2456] : nr_iowait_cpu(): : } 0.00 : ffff800010117f94: sxtw x0, w0 0.00 : ffff800010117f98: ret Percent | Source code & Disassembly of vmlinux for cycles (5 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011ac68 : : calc_load_fold_active(): : : long calc_load_fold_active(struct rq *this_rq, long adjust) : { : long nr_active, delta = 0; : : nr_active = this_rq->nr_running - adjust; 100.00 : ffff80001011ac68: ldr w2, [x0, #4] : nr_active += (long)this_rq->nr_uninterruptible; 0.00 : ffff80001011ac6c: ldr x3, [x0, #2344] : : if (nr_active != this_rq->calc_load_active) { 0.00 : ffff80001011ac70: ldr x1, [x0, #2864] : nr_active += (long)this_rq->nr_uninterruptible; 0.00 : ffff80001011ac74: add x2, x2, x3 : if (nr_active != this_rq->calc_load_active) { 0.00 : ffff80001011ac78: cmp x2, x1 0.00 : ffff80001011ac7c: b.eq ffff80001011accc // b.none : calc_load_write_idx(): : static atomic_long_t calc_load_nohz[2]; : static int calc_load_idx; : : static inline int calc_load_write_idx(void) : { : int idx = calc_load_idx; 0.00 : ffff80001011ac80: adrp x3, ffff800011afb000 : calc_load_fold_active(): : delta = nr_active - this_rq->calc_load_active; 0.00 : ffff80001011ac84: sub x1, x2, x1 : calc_load_write_idx(): : int idx = calc_load_idx; 0.00 : ffff80001011ac88: add x4, x3, #0xf48 : calc_load_fold_active(): : this_rq->calc_load_active = nr_active; 0.00 : ffff80001011ac8c: str x2, [x0, #2864] : calc_load_write_idx(): : int idx = calc_load_idx; 0.00 : ffff80001011ac90: ldr w3, [x3, #3912] : : /* : * See calc_global_nohz(), if we observe the new index, we also : * need to observe the new update time. : */ : smp_rmb(); 0.00 : ffff80001011ac94: dmb ishld : : /* : * If the folding window started, make sure we start writing in the : * next NO_HZ-delta. : */ : if (!time_before(jiffies, READ_ONCE(calc_load_update))) 0.00 : ffff80001011ac98: adrp x2, ffff800011907000 : calc_load_nohz_fold(): : : delta = calc_load_fold_active(rq, 0); : if (delta) { : int idx = calc_load_write_idx(); : : atomic_long_add(delta, &calc_load_nohz[idx]); 0.00 : ffff80001011ac9c: add x0, x4, #0x10 : calc_load_write_idx(): : if (!time_before(jiffies, READ_ONCE(calc_load_update))) 0.00 : ffff80001011aca0: ldr x2, [x2, #2432] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011aca4: ldr x4, [x4, #8] : calc_load_write_idx(): 0.00 : ffff80001011aca8: sub x2, x2, x4 : idx++; 0.00 : ffff80001011acac: mvn x2, x2 0.00 : ffff80001011acb0: lsr x2, x2, #63 0.00 : ffff80001011acb4: add w2, w3, w2 : calc_load_nohz_fold(): : atomic_long_add(delta, &calc_load_nohz[idx]); 0.00 : ffff80001011acb8: ubfiz x2, x2, #3, #1 0.00 : ffff80001011acbc: add x0, x0, x2 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011acc0: b ffff80001011acd0 0.00 : ffff80001011acc4: b ffff80001011acd0 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff80001011acc8: stadd x1, [x0] : calc_load_nohz_fold(): : } : } 0.00 : ffff80001011accc: ret : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff80001011acd0: b ffff80001011b0d0 : calc_load_nohz_fold(): 0.00 : ffff80001011acd4: ret Percent | Source code & Disassembly of vmlinux for cycles (5 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001048da10 : : find_next_and_bit(): : : #if !defined(find_next_and_bit) : unsigned long find_next_and_bit(const unsigned long *addr1, : const unsigned long *addr2, unsigned long size, : unsigned long offset) : { 100.00 : ffff80001048da10: stp x29, x30, [sp, #-16]! : return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); 0.00 : ffff80001048da14: mov x4, #0x0 // #0 : { 0.00 : ffff80001048da18: mov x29, sp : return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); 0.00 : ffff80001048da1c: bl ffff80001048d988 <_find_next_bit.constprop.1> : } 0.00 : ffff80001048da20: ldp x29, x30, [sp], #16 0.00 : ffff80001048da24: ret Percent | Source code & Disassembly of vmlinux for cycles (19 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101209d0 : : put_prev_entity(): : } : : static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); : : static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) : { 63.44 : ffff8000101209d0: stp x29, x30, [sp, #-32]! 0.00 : ffff8000101209d4: mov x29, sp 0.00 : ffff8000101209d8: stp x19, x20, [sp, #16] 0.00 : ffff8000101209dc: mov x19, x1 0.00 : ffff8000101209e0: mov x20, x0 : /* : * If still on the runqueue then deactivate_task() : * was not called and update_curr() has to be done: : */ : if (prev->on_rq) 5.27 : ffff8000101209e4: ldr w1, [x1, #64] 0.00 : ffff8000101209e8: cbnz w1, ffff8000101209fc : /* Put 'current' back into the tree. */ : __enqueue_entity(cfs_rq, prev); : /* in !on_rq case, update occurred at dequeue */ : update_load_avg(cfs_rq, prev, 0); : } : cfs_rq->curr = NULL; 0.00 : ffff8000101209ec: str xzr, [x20, #72] : } 0.00 : ffff8000101209f0: ldp x19, x20, [sp, #16] 0.00 : ffff8000101209f4: ldp x29, x30, [sp], #32 0.00 : ffff8000101209f8: ret : update_curr(cfs_rq); 15.65 : ffff8000101209fc: bl ffff80001011fab8 : if (prev->on_rq) { 0.00 : ffff800010120a00: ldr w0, [x19, #64] 0.00 : ffff800010120a04: cbz w0, ffff8000101209ec : __enqueue_entity(cfs_rq, prev); 0.00 : ffff800010120a08: mov x1, x19 0.00 : ffff800010120a0c: mov x0, x20 0.00 : ffff800010120a10: bl ffff80001011ce78 <__enqueue_entity> : update_load_avg(cfs_rq, prev, 0); 10.70 : ffff800010120a14: mov x1, x19 0.00 : ffff800010120a18: mov x0, x20 0.00 : ffff800010120a1c: mov w2, #0x0 // #0 0.00 : ffff800010120a20: bl ffff80001011e940 : cfs_rq->curr = NULL; 0.00 : ffff800010120a24: str xzr, [x20, #72] : } 4.94 : ffff800010120a28: ldp x19, x20, [sp, #16] 0.00 : ffff800010120a2c: ldp x29, x30, [sp], #32 0.00 : ffff800010120a30: ret Percent | Source code & Disassembly of vmlinux for cycles (8 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101d7528 : : mempool_alloc_slab(): : : /* : * A commonly used alloc and free fn. : */ : void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) : { 38.03 : ffff8000101d7528: stp x29, x30, [sp, #-16]! 0.00 : ffff8000101d752c: mov x2, x1 : struct kmem_cache *mem = pool_data; : VM_BUG_ON(mem->ctor); : return kmem_cache_alloc(mem, gfp_mask); 0.00 : ffff8000101d7530: mov w1, w0 0.00 : ffff8000101d7534: mov x0, x2 : { 37.93 : ffff8000101d7538: mov x29, sp : return kmem_cache_alloc(mem, gfp_mask); 0.00 : ffff8000101d753c: bl ffff800010252238 : } 24.04 : ffff8000101d7540: ldp x29, x30, [sp], #16 0.00 : ffff8000101d7544: ret Percent | Source code & Disassembly of vmlinux for cycles (14 samples, percent: local period) --------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104645b8 : : hctx_lock(): : srcu_read_unlock(hctx->srcu, srcu_idx); : } : : static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx) : __acquires(hctx->srcu) : { 0.00 : ffff8000104645b8: stp x29, x30, [sp, #-32]! 0.00 : ffff8000104645bc: mov x29, sp 0.00 : ffff8000104645c0: str x19, [sp, #16] 0.00 : ffff8000104645c4: mov x19, x1 : if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { 20.83 : ffff8000104645c8: ldr x1, [x0, #192] 0.00 : ffff8000104645cc: tbz w1, #5, ffff8000104645e8 : srcu_read_lock(): : */ : static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp) : { : int retval; : : retval = __srcu_read_lock(ssp); 0.00 : ffff8000104645d0: add x0, x0, #0x240 0.00 : ffff8000104645d4: bl ffff80001015b3d8 <__srcu_read_lock> : hctx_lock(): : /* shut up gcc false positive */ : *srcu_idx = 0; : rcu_read_lock(); : } else : *srcu_idx = srcu_read_lock(hctx->srcu); 0.00 : ffff8000104645d8: str w0, [x19] : } 0.00 : ffff8000104645dc: ldr x19, [sp, #16] 0.00 : ffff8000104645e0: ldp x29, x30, [sp], #32 0.00 : ffff8000104645e4: ret : *srcu_idx = 0; 72.05 : ffff8000104645e8: str wzr, [x19] : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff8000104645ec: bl ffff80001015c1b0 <__rcu_read_lock> : hctx_lock(): : } 7.12 : ffff8000104645f0: ldr x19, [sp, #16] 0.00 : ffff8000104645f4: ldp x29, x30, [sp], #32 0.00 : ffff8000104645f8: ret Percent | Source code & Disassembly of vmlinux for cycles (6 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011caa8 <__calc_delta>: : __update_inv_weight(): : : static void __update_inv_weight(struct load_weight *lw) : { : unsigned long w; : : if (likely(lw->inv_weight)) 0.00 : ffff80001011caa8: ldr w4, [x2, #8] : __calc_delta(): : * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus : * weight/lw.weight <= 1, and therefore our shift will also be positive. : */ : static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) : { : u64 fact = scale_load_down(weight); 0.00 : ffff80001011caac: lsr x3, x1, #10 : __update_inv_weight(): : if (likely(lw->inv_weight)) 0.00 : ffff80001011cab0: cbz w4, ffff80001011cb48 <__calc_delta+0xa0> 17.28 : ffff80001011cab4: mov w4, w4 : __calc_delta(): : int shift = WMULT_SHIFT; 0.00 : ffff80001011cab8: mov w2, #0x20 // #32 : : __update_inv_weight(lw); : : if (unlikely(fact >> 32)) { 0.00 : ffff80001011cabc: negs xzr, x1, lsr #42 0.00 : ffff80001011cac0: b.ne ffff80001011cb28 <__calc_delta+0x80> // b.any : mul_u32_u32(): : /* : * Many a GCC version messes this up and generates a 64x64 mult :-( : */ : static inline u64 mul_u32_u32(u32 a, u32 b) : { : return (u64)a * b; 16.01 : ffff80001011cac4: and x1, x3, #0xffffffff 0.00 : ffff80001011cac8: mul x1, x1, x4 : __calc_delta(): : } : } : : fact = mul_u32_u32(fact, lw->inv_weight); : : while (fact >> 32) { 0.00 : ffff80001011cacc: negs xzr, x1, lsr #32 0.00 : ffff80001011cad0: b.eq ffff80001011cb18 <__calc_delta+0x70> // b.none 0.00 : ffff80001011cad4: nop : fact >>= 1; 0.00 : ffff80001011cad8: lsr x3, x1, #1 : while (fact >> 32) { 0.00 : ffff80001011cadc: lsr x4, x1, #33 : shift--; 16.16 : ffff80001011cae0: sub w2, w2, #0x1 : fact >>= 1; 0.00 : ffff80001011cae4: mov x1, x3 : while (fact >> 32) { 17.89 : ffff80001011cae8: cbnz x4, ffff80001011cad8 <__calc_delta+0x30> : mul_u64_u32_shr(): : #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) : : #ifndef mul_u64_u32_shr : static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) : { : return (u64)(((unsigned __int128)a * mul) >> shift); 0.00 : ffff80001011caec: umulh x1, x0, x3 0.00 : ffff80001011caf0: mvn w4, w2 0.00 : ffff80001011caf4: mul x0, x0, x3 0.00 : ffff80001011caf8: tst x2, #0x40 17.52 : ffff80001011cafc: lsl x3, x1, #1 0.00 : ffff80001011cb00: lsr x0, x0, x2 0.00 : ffff80001011cb04: lsr x2, x1, x2 0.00 : ffff80001011cb08: lsl x1, x3, x4 0.00 : ffff80001011cb0c: orr x0, x1, x0 0.00 : ffff80001011cb10: csel x0, x0, x2, eq // eq = none : __calc_delta(): : } : : return mul_u64_u32_shr(delta_exec, fact, shift); : } 0.00 : ffff80001011cb14: ret : mul_u32_u32(): : return (u64)a * b; 0.00 : ffff80001011cb18: mov x3, x1 0.00 : ffff80001011cb1c: b ffff80001011caec <__calc_delta+0x44> : __calc_delta(): : if (unlikely(fact >> 32)) { 0.00 : ffff80001011cb20: mov x4, #0x1 // #1 0.00 : ffff80001011cb24: nop 0.00 : ffff80001011cb28: mov x1, x3 0.00 : ffff80001011cb2c: mov w2, #0x20 // #32 : fact >>= 1; 0.00 : ffff80001011cb30: lsr x3, x3, #1 : while (fact >> 32) { 0.00 : ffff80001011cb34: lsr x5, x1, #33 : shift--; 0.00 : ffff80001011cb38: sub w2, w2, #0x1 : fact >>= 1; 0.00 : ffff80001011cb3c: mov x1, x3 : while (fact >> 32) { 0.00 : ffff80001011cb40: cbnz x5, ffff80001011cb30 <__calc_delta+0x88> 0.00 : ffff80001011cb44: b ffff80001011cac4 <__calc_delta+0x1c> : __update_inv_weight(): : w = scale_load_down(lw->weight); 0.00 : ffff80001011cb48: ldr x4, [x2] : if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 0.00 : ffff80001011cb4c: mov x6, #0xfffffffe // #4294967294 : w = scale_load_down(lw->weight); 0.00 : ffff80001011cb50: lsr x5, x4, #10 : if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 0.00 : ffff80001011cb54: cmp x5, x6 15.14 : ffff80001011cb58: b.hi ffff80001011cb70 <__calc_delta+0xc8> // b.pmore : else if (unlikely(!w)) 0.00 : ffff80001011cb5c: cbz x5, ffff80001011cb8c <__calc_delta+0xe4> : lw->inv_weight = WMULT_CONST / w; 0.00 : ffff80001011cb60: mov x4, #0xffffffff // #4294967295 0.00 : ffff80001011cb64: udiv x4, x4, x5 0.00 : ffff80001011cb68: str w4, [x2, #8] 0.00 : ffff80001011cb6c: b ffff80001011cab8 <__calc_delta+0x10> : lw->inv_weight = 1; 0.00 : ffff80001011cb70: mov w4, #0x1 // #1 0.00 : ffff80001011cb74: str w4, [x2, #8] : __calc_delta(): : if (unlikely(fact >> 32)) { 0.00 : ffff80001011cb78: negs xzr, x1, lsr #42 0.00 : ffff80001011cb7c: b.ne ffff80001011cb20 <__calc_delta+0x78> // b.any 0.00 : ffff80001011cb80: and x3, x3, #0xffffffff : int shift = WMULT_SHIFT; 0.00 : ffff80001011cb84: mov w2, #0x20 // #32 0.00 : ffff80001011cb88: b ffff80001011caec <__calc_delta+0x44> : __update_inv_weight(): : lw->inv_weight = WMULT_CONST; 0.00 : ffff80001011cb8c: mov w5, #0xffffffff // #-1 0.00 : ffff80001011cb90: mov x4, #0xffffffff // #4294967295 0.00 : ffff80001011cb94: str w5, [x2, #8] 0.00 : ffff80001011cb98: b ffff80001011cab8 <__calc_delta+0x10> Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015ee98 <__note_gp_changes>: : __note_gp_changes(): : const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) && : rcu_segcblist_is_offloaded(&rdp->cblist); : : raw_lockdep_assert_held_rcu_node(rnp); : : if (rdp->gp_seq == rnp->gp_seq) 0.00 : ffff80001015ee98: ldr x3, [x0, #8] 24.20 : ffff80001015ee9c: ldr x2, [x1] 0.00 : ffff80001015eea0: cmp x2, x3 0.00 : ffff80001015eea4: b.eq ffff80001015effc <__note_gp_changes+0x164> // b.none : { 0.00 : ffff80001015eea8: stp x29, x30, [sp, #-32]! : rcu_seq_completed_gp(): : /* : * Has a grace period completed since the time the old gp_seq was collected? : */ : static inline bool rcu_seq_completed_gp(unsigned long old, unsigned long new) : { : return ULONG_CMP_LT(old, new & ~RCU_SEQ_STATE_MASK); 0.00 : ffff80001015eeac: and x3, x3, #0xfffffffffffffffc 0.00 : ffff80001015eeb0: sub x2, x2, x3 : __note_gp_changes(): 0.00 : ffff80001015eeb4: mov x29, sp 0.00 : ffff80001015eeb8: stp x19, x20, [sp, #16] 0.00 : ffff80001015eebc: mov x20, x0 0.00 : ffff80001015eec0: mov x19, x1 : return false; /* Nothing to do. */ : : /* Handle the ends of any preceding grace periods first. */ : if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) || 0.00 : ffff80001015eec4: tbnz x2, #63, ffff80001015ef74 <__note_gp_changes+0xdc> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001015eec8: ldrb w2, [x1, #20] : __note_gp_changes(): 0.00 : ffff80001015eecc: tst w2, #0xff 0.00 : ffff80001015eed0: b.ne ffff80001015ef74 <__note_gp_changes+0xdc> // b.any : if (!offloaded) : ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */ : trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); : } else { : if (!offloaded) : ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */ 0.00 : ffff80001015eed4: bl ffff80001015c9d8 0.00 : ffff80001015eed8: and w0, w0, #0xff : rcu_seq_new_gp(): : /* : * Has a grace period started since the time the old gp_seq was collected? : */ : static inline bool rcu_seq_new_gp(unsigned long old, unsigned long new) : { : return ULONG_CMP_LT((old + RCU_SEQ_STATE_MASK) & ~RCU_SEQ_STATE_MASK, 0.00 : ffff80001015eedc: ldr x1, [x19] : __note_gp_changes(): : } : : /* Now handle the beginnings of any new-to-this-CPU grace periods. */ : if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) || 0.00 : ffff80001015eee0: ldr x2, [x20, #8] : rcu_seq_new_gp(): 0.00 : ffff80001015eee4: add x1, x1, #0x3 0.00 : ffff80001015eee8: and x1, x1, #0xfffffffffffffffc 0.00 : ffff80001015eeec: sub x1, x1, x2 : __note_gp_changes(): 0.00 : ffff80001015eef0: tbnz x1, #63, ffff80001015ef9c <__note_gp_changes+0x104> : __read_once_size(): 0.00 : ffff80001015eef4: ldrb w1, [x19, #20] : __note_gp_changes(): 0.00 : ffff80001015eef8: tst w1, #0xff 0.00 : ffff80001015eefc: b.ne ffff80001015ef9c <__note_gp_changes+0x104> // b.any : need_gp = !!(rnp->qsmask & rdp->grpmask); : rdp->cpu_no_qs.b.norm = need_gp; : rdp->core_needs_qs = need_gp; : zero_cpu_stall_ticks(rdp); : } : rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */ 24.92 : ffff80001015ef00: str x2, [x19] : if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap) 0.00 : ffff80001015ef04: ldr x1, [x19, #8] 0.00 : ffff80001015ef08: ldr x2, [x20, #16] 0.00 : ffff80001015ef0c: sub x1, x1, x2 0.00 : ffff80001015ef10: tbnz x1, #63, ffff80001015ef6c <__note_gp_changes+0xd4> 0.00 : ffff80001015ef14: ldrb w1, [x19, #20] 0.00 : ffff80001015ef18: cbnz w1, ffff80001015ef6c <__note_gp_changes+0xd4> : __write_once_size(): : } : : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff80001015ef1c: strb wzr, [x19, #20] : rcu_gpnum_ovf(): : if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4, 0.00 : ffff80001015ef20: mov x3, #0x3fffffffffffffff // #4611686018427387903 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001015ef24: ldr x1, [x19] : rcu_gpnum_ovf(): 0.00 : ffff80001015ef28: ldr x2, [x20, #8] 0.00 : ffff80001015ef2c: sub x1, x1, x2 0.00 : ffff80001015ef30: cmn x1, x3 0.00 : ffff80001015ef34: b.pl ffff80001015ef44 <__note_gp_changes+0xac> // b.nfrst : __write_once_size(): : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; 0.00 : ffff80001015ef38: mov w1, #0x1 // #1 0.00 : ffff80001015ef3c: strb w1, [x19, #20] 0.00 : ffff80001015ef40: ldr x2, [x20, #8] : rcu_gpnum_ovf(): : if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq)) 0.00 : ffff80001015ef44: ldr x1, [x19, #304] 0.00 : ffff80001015ef48: mov x3, #0x3fffffffffffffff // #4611686018427387903 0.00 : ffff80001015ef4c: add x1, x1, x3 0.00 : ffff80001015ef50: sub x1, x1, x2 0.00 : ffff80001015ef54: tbz x1, #63, ffff80001015ef60 <__note_gp_changes+0xc8> : rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4; 0.00 : ffff80001015ef58: add x2, x2, x3 0.00 : ffff80001015ef5c: str x2, [x19, #304] : __note_gp_changes(): : rdp->gp_seq_needed = rnp->gp_seq_needed; : WRITE_ONCE(rdp->gpwrap, false); : rcu_gpnum_ovf(rnp, rdp); : return ret; : } 0.00 : ffff80001015ef60: ldp x19, x20, [sp, #16] 0.00 : ffff80001015ef64: ldp x29, x30, [sp], #32 0.00 : ffff80001015ef68: ret : rdp->gp_seq_needed = rnp->gp_seq_needed; 0.00 : ffff80001015ef6c: str x2, [x19, #8] 0.00 : ffff80001015ef70: b ffff80001015ef1c <__note_gp_changes+0x84> : ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */ 26.46 : ffff80001015ef74: mov x1, x19 0.00 : ffff80001015ef78: mov x0, x20 0.00 : ffff80001015ef7c: bl ffff80001015cbb8 0.00 : ffff80001015ef80: and w0, w0, #0xff : rcu_seq_new_gp(): 0.00 : ffff80001015ef84: ldr x1, [x19] : __note_gp_changes(): : if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) || 0.00 : ffff80001015ef88: ldr x2, [x20, #8] : rcu_seq_new_gp(): 0.00 : ffff80001015ef8c: add x1, x1, #0x3 0.00 : ffff80001015ef90: and x1, x1, #0xfffffffffffffffc 0.00 : ffff80001015ef94: sub x1, x1, x2 : __note_gp_changes(): 0.00 : ffff80001015ef98: tbz x1, #63, ffff80001015eef4 <__note_gp_changes+0x5c> : need_gp = !!(rnp->qsmask & rdp->grpmask); 0.00 : ffff80001015ef9c: ldr x3, [x20, #32] : zero_cpu_stall_ticks(): : : /* Zero ->ticks_this_gp and snapshot the number of RCU softirq handlers. */ : static void zero_cpu_stall_ticks(struct rcu_data *rdp) : { : rdp->ticks_this_gp = 0; : rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id()); 0.00 : ffff80001015efa0: adrp x1, ffff80001151d000 : __note_gp_changes(): 0.00 : ffff80001015efa4: ldr x2, [x19, #32] : zero_cpu_stall_ticks(): 0.00 : ffff80001015efa8: add x1, x1, #0x18 : rdp->ticks_this_gp = 0; 0.00 : ffff80001015efac: str xzr, [x19, #40] : __note_gp_changes(): 0.00 : ffff80001015efb0: tst x3, x2 : zero_cpu_stall_ticks(): : WRITE_ONCE(rdp->last_fqs_resched, jiffies); 0.00 : ffff80001015efb4: adrp x3, ffff800011907000 : __note_gp_changes(): 0.00 : ffff80001015efb8: cset w2, ne // ne = any : rdp->cpu_no_qs.b.norm = need_gp; 0.00 : ffff80001015efbc: strb w2, [x19, #16] : rdp->core_needs_qs = need_gp; 24.42 : ffff80001015efc0: strb w2, [x19, #18] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001015efc4: mrs x2, tpidr_el1 : kstat_softirqs_cpu(): : __this_cpu_inc(kstat.softirqs[irq]); : } : : static inline unsigned int kstat_softirqs_cpu(unsigned int irq, int cpu) : { : return kstat_cpu(cpu).softirqs[irq]; 0.00 : ffff80001015efc8: ldrsw x4, [x1, x2] 0.00 : ffff80001015efcc: adrp x2, ffff800011909000 0.00 : ffff80001015efd0: add x2, x2, #0x928 0.00 : ffff80001015efd4: adrp x1, ffff80001151f000 0.00 : ffff80001015efd8: add x1, x1, #0x10 : zero_cpu_stall_ticks(): 0.00 : ffff80001015efdc: ldr x3, [x3, #2432] : kstat_softirqs_cpu(): 0.00 : ffff80001015efe0: ldr x2, [x2, x4, lsl #3] 0.00 : ffff80001015efe4: add x1, x2, x1 0.00 : ffff80001015efe8: ldr w1, [x1, #44] : zero_cpu_stall_ticks(): : rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id()); 0.00 : ffff80001015efec: str w1, [x19, #264] : __write_once_size(): : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001015eff0: str x3, [x19, #344] 0.00 : ffff80001015eff4: ldr x2, [x20, #8] 0.00 : ffff80001015eff8: b ffff80001015ef00 <__note_gp_changes+0x68> : __note_gp_changes(): : return false; /* Nothing to do. */ 0.00 : ffff80001015effc: mov w0, #0x0 // #0 : } 0.00 : ffff80001015f000: ret Percent | Source code & Disassembly of vmlinux for cycles (6 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011c438 : : update_min_vruntime(): : return (s64)(a->vruntime - b->vruntime) < 0; : } : : static void update_min_vruntime(struct cfs_rq *cfs_rq) : { : struct sched_entity *curr = cfs_rq->curr; 0.00 : ffff80001011c438: ldp x3, x2, [x0, #64] : struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline); : : u64 vruntime = cfs_rq->min_vruntime; 15.27 : ffff80001011c43c: ldr x1, [x0, #48] : : if (curr) { 0.00 : ffff80001011c440: cbz x2, ffff80001011c484 : if (curr->on_rq) 17.51 : ffff80001011c444: ldr w4, [x2, #64] 0.00 : ffff80001011c448: cbz w4, ffff80001011c478 : vruntime = curr->vruntime; 0.00 : ffff80001011c44c: ldr x2, [x2, #88] : else : curr = NULL; : } : : if (leftmost) { /* non-empty tree */ 0.00 : ffff80001011c450: cbz x3, ffff80001011c464 : se = rb_entry(leftmost, struct sched_entity, run_node); : : if (!curr) : vruntime = se->vruntime; : else : vruntime = min_vruntime(vruntime, se->vruntime); 0.00 : ffff80001011c454: ldr x3, [x3, #64] : min_vruntime(): : s64 delta = (s64)(vruntime - min_vruntime); 0.00 : ffff80001011c458: sub x4, x3, x2 : min_vruntime = vruntime; 0.00 : ffff80001011c45c: cmp x4, #0x0 0.00 : ffff80001011c460: csel x2, x2, x3, ge // ge = tcont : max_vruntime(): : s64 delta = (s64)(vruntime - max_vruntime); 67.22 : ffff80001011c464: sub x3, x2, x1 : if (delta > 0) 0.00 : ffff80001011c468: cmp x3, #0x0 0.00 : ffff80001011c46c: csel x1, x1, x2, le : update_min_vruntime(): : } : : /* ensure we never gain time by being placed backwards. */ : cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); 0.00 : ffff80001011c470: str x1, [x0, #48] : #ifndef CONFIG_64BIT : smp_wmb(); : cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; : #endif : } 0.00 : ffff80001011c474: ret : if (leftmost) { /* non-empty tree */ 0.00 : ffff80001011c478: cbz x3, ffff80001011c470 : vruntime = se->vruntime; 0.00 : ffff80001011c47c: ldr x2, [x3, #64] 0.00 : ffff80001011c480: b ffff80001011c464 : if (leftmost) { /* non-empty tree */ 0.00 : ffff80001011c484: cbnz x3, ffff80001011c47c : cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); 0.00 : ffff80001011c488: str x1, [x0, #48] : } 0.00 : ffff80001011c48c: ret Percent | Source code & Disassembly of vmlinux for cycles (7 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101f3c60 : : next_online_pgdat(): : { : return NODE_DATA(first_online_node); : } : : struct pglist_data *next_online_pgdat(struct pglist_data *pgdat) : { 13.60 : ffff8000101f3c60: stp x29, x30, [sp, #-16]! : __next_node(): : } : : #define next_node(n, src) __next_node((n), &(src)) : static inline int __next_node(int n, const nodemask_t *srcp) : { : return min_t(int,MAX_NUMNODES,find_next_bit(srcp->bits, MAX_NUMNODES, n+1)); 0.00 : ffff8000101f3c64: mov x1, #0x4 // #4 : next_online_pgdat(): 0.00 : ffff8000101f3c68: mov x29, sp : __next_node(): 0.00 : ffff8000101f3c6c: ldr w2, [x0, #7232] 0.00 : ffff8000101f3c70: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101f3c74: add x0, x0, #0x428 0.00 : ffff8000101f3c78: add w2, w2, #0x1 29.27 : ffff8000101f3c7c: sxtw x2, w2 0.00 : ffff8000101f3c80: bl ffff80001048daa8 0.00 : ffff8000101f3c84: cmp w0, #0x4 0.00 : ffff8000101f3c88: mov w1, #0x4 // #4 0.00 : ffff8000101f3c8c: csel w1, w0, w1, le : next_online_pgdat(): : int nid = next_online_node(pgdat->node_id); : : if (nid == MAX_NUMNODES) 0.00 : ffff8000101f3c90: cmp w0, #0x3 14.95 : ffff8000101f3c94: b.gt ffff8000101f3cac : return NULL; : return NODE_DATA(nid); 0.00 : ffff8000101f3c98: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff8000101f3c9c: add x0, x0, #0x128 28.61 : ffff8000101f3ca0: ldr x0, [x0, w1, sxtw #3] : } 13.55 : ffff8000101f3ca4: ldp x29, x30, [sp], #16 0.00 : ffff8000101f3ca8: ret : return NULL; 0.00 : ffff8000101f3cac: mov x0, #0x0 // #0 0.00 : ffff8000101f3cb0: b ffff8000101f3ca4 Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011bf30 : : arch_cpu_idle_enter(): : : return 1; : } : : /* Weak implementations for optional arch specific functions */ : void __weak arch_cpu_idle_prepare(void) { } 100.00 : ffff80001011bf30: ret Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000105323e0 : : pci_generic_config_read(): : EXPORT_SYMBOL(pci_bus_write_config_word); : EXPORT_SYMBOL(pci_bus_write_config_dword); : : int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, : int where, int size, u32 *val) : { 0.00 : ffff8000105323e0: stp x29, x30, [sp, #-32]! 0.00 : ffff8000105323e4: mov x29, sp 0.00 : ffff8000105323e8: stp x19, x20, [sp, #16] 0.00 : ffff8000105323ec: mov w19, w3 0.00 : ffff8000105323f0: mov x20, x4 : void __iomem *addr; : : addr = bus->ops->map_bus(bus, devfn, where); 0.00 : ffff8000105323f4: ldr x5, [x0, #192] 0.00 : ffff8000105323f8: ldr x3, [x5, #16] 0.00 : ffff8000105323fc: blr x3 : if (!addr) { 0.00 : ffff800010532400: cbz x0, ffff80001053243c : *val = ~0; : return PCIBIOS_DEVICE_NOT_FOUND; : } : : if (size == 1) 0.00 : ffff800010532404: cmp w19, #0x1 0.00 : ffff800010532408: b.eq ffff800010532454 // b.none : *val = readb(addr); : else if (size == 2) 0.00 : ffff80001053240c: cmp w19, #0x2 0.00 : ffff800010532410: b.eq ffff800010532480 // b.none : __raw_readl(): : : #define __raw_readl __raw_readl : static __always_inline u32 __raw_readl(const volatile void __iomem *addr) : { : u32 val; : asm volatile(ALTERNATIVE("ldr %w0, [%1]", 0.00 : ffff800010532414: ldr w1, [x0] : pci_generic_config_read(): : *val = readw(addr); : else : *val = readl(addr); 0.00 : ffff800010532418: dmb oshld 0.00 : ffff80001053241c: mov w0, w1 0.00 : ffff800010532420: eor x0, x0, x0 0.00 : ffff800010532424: cbnz x0, ffff800010532424 0.00 : ffff800010532428: str w1, [x20] : : return PCIBIOS_SUCCESSFUL; 0.00 : ffff80001053242c: mov w0, #0x0 // #0 : } 0.00 : ffff800010532430: ldp x19, x20, [sp, #16] 0.00 : ffff800010532434: ldp x29, x30, [sp], #32 0.00 : ffff800010532438: ret : *val = ~0; 0.00 : ffff80001053243c: mov w1, #0xffffffff // #-1 0.00 : ffff800010532440: str w1, [x20] : return PCIBIOS_DEVICE_NOT_FOUND; 0.00 : ffff800010532444: mov w0, #0x86 // #134 : } 0.00 : ffff800010532448: ldp x19, x20, [sp, #16] 0.00 : ffff80001053244c: ldp x29, x30, [sp], #32 0.00 : ffff800010532450: ret : __raw_readb(): : asm volatile(ALTERNATIVE("ldrb %w0, [%1]", 0.00 : ffff800010532454: ldrb w0, [x0] 0.00 : ffff800010532458: and w1, w0, #0xff : pci_generic_config_read(): : *val = readb(addr); 0.00 : ffff80001053245c: dmb oshld 0.00 : ffff800010532460: and x0, x0, #0xff 0.00 : ffff800010532464: eor x0, x0, x0 0.00 : ffff800010532468: cbnz x0, ffff800010532468 0.00 : ffff80001053246c: str w1, [x20] : return PCIBIOS_SUCCESSFUL; 0.00 : ffff800010532470: mov w0, #0x0 // #0 : } 0.00 : ffff800010532474: ldp x19, x20, [sp, #16] 0.00 : ffff800010532478: ldp x29, x30, [sp], #32 0.00 : ffff80001053247c: ret : __raw_readw(): : asm volatile(ALTERNATIVE("ldrh %w0, [%1]", 0.00 : ffff800010532480: ldrh w0, [x0] 0.00 : ffff800010532484: and w1, w0, #0xffff : pci_generic_config_read(): : *val = readw(addr); 27.14 : ffff800010532488: dmb oshld 72.86 : ffff80001053248c: and x0, x0, #0xffff 0.00 : ffff800010532490: eor x0, x0, x0 0.00 : ffff800010532494: cbnz x0, ffff800010532494 0.00 : ffff800010532498: str w1, [x20] : return PCIBIOS_SUCCESSFUL; 0.00 : ffff80001053249c: mov w0, #0x0 // #0 : } 0.00 : ffff8000105324a0: ldp x19, x20, [sp, #16] 0.00 : ffff8000105324a4: ldp x29, x30, [sp], #32 0.00 : ffff8000105324a8: ret Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016bad0 : : hrtimer_cancel(): : * Returns: : * 0 when the timer was not active : * 1 when the timer was active : */ : int hrtimer_cancel(struct hrtimer *timer) : { 0.00 : ffff80001016bad0: stp x29, x30, [sp, #-32]! 0.00 : ffff80001016bad4: mov x29, sp 75.63 : ffff80001016bad8: str x19, [sp, #16] 0.00 : ffff80001016badc: mov x19, x0 : int ret; : : do { : ret = hrtimer_try_to_cancel(timer); 0.00 : ffff80001016bae0: mov x0, x19 0.00 : ffff80001016bae4: bl ffff80001016b9f8 : : if (ret < 0) 0.00 : ffff80001016bae8: tbnz w0, #31, ffff80001016baf8 : hrtimer_cancel_wait_running(timer); : } while (ret < 0); : return ret; : } 24.37 : ffff80001016baec: ldr x19, [sp, #16] 0.00 : ffff80001016baf0: ldp x29, x30, [sp], #32 0.00 : ffff80001016baf4: ret : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff80001016baf8: yield 0.00 : ffff80001016bafc: b ffff80001016bae0 Percent | Source code & Disassembly of vmlinux for cycles (7 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101f3cb8 : : next_zone(): : /* : * next_zone - helper magic for for_each_zone() : */ : struct zone *next_zone(struct zone *zone) : { : pg_data_t *pgdat = zone->zone_pgdat; 27.19 : ffff8000101f3cb8: ldr x1, [x0, #80] : : if (zone < pgdat->node_zones + MAX_NR_ZONES - 1) 0.00 : ffff8000101f3cbc: mov x2, #0x1380 // #4992 0.00 : ffff8000101f3cc0: add x2, x1, x2 0.00 : ffff8000101f3cc4: cmp x0, x2 57.77 : ffff8000101f3cc8: b.cs ffff8000101f3cd4 // b.hs, b.nlast : zone++; 0.00 : ffff8000101f3ccc: add x0, x0, #0x680 : zone = pgdat->node_zones; : else : zone = NULL; : } : return zone; : } 0.00 : ffff8000101f3cd0: ret : { 15.04 : ffff8000101f3cd4: stp x29, x30, [sp, #-16]! : pgdat = next_online_pgdat(pgdat); 0.00 : ffff8000101f3cd8: mov x0, x1 : { 0.00 : ffff8000101f3cdc: mov x29, sp : pgdat = next_online_pgdat(pgdat); 0.00 : ffff8000101f3ce0: bl ffff8000101f3c60 : } 0.00 : ffff8000101f3ce4: ldp x29, x30, [sp], #16 0.00 : ffff8000101f3ce8: ret Percent | Source code & Disassembly of vmlinux for cycles (8 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101973e8 <__cgroup_account_cputime>: : __cgroup_account_cputime(): : cgroup_rstat_updated(cgrp, smp_processor_id()); : put_cpu_ptr(rstatc); : } : : void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec) : { 12.50 : ffff8000101973e8: stp x29, x30, [sp, #-16]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000101973ec: mrs x3, sp_el0 : __cgroup_account_cputime(): 0.00 : ffff8000101973f0: mov x29, sp : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101973f4: ldr w2, [x3, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000101973f8: add w2, w2, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101973fc: str w2, [x3, #16] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010197400: mrs x3, tpidr_el1 : cgroup_base_stat_cputime_account_begin(): : rstatc = get_cpu_ptr(cgrp->rstat_cpu); 0.00 : ffff800010197404: ldr x2, [x0, #672] 0.00 : ffff800010197408: add x2, x2, x3 : __cgroup_account_cputime(): : struct cgroup_rstat_cpu *rstatc; : : rstatc = cgroup_base_stat_cputime_account_begin(cgrp); : rstatc->bstat.cputime.sum_exec_runtime += delta_exec; 0.00 : ffff80001019740c: ldr x3, [x2, #16] 0.00 : ffff800010197410: add x1, x3, x1 87.50 : ffff800010197414: str x1, [x2, #16] : cgroup_base_stat_cputime_account_end(cgrp, rstatc); 0.00 : ffff800010197418: bl ffff800010197170 : } 0.00 : ffff80001019741c: ldp x29, x30, [sp], #16 0.00 : ffff800010197420: ret Percent | Source code & Disassembly of vmlinux for cycles (8 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010253890 : : kfree(): : struct page *page; : void *object = (void *)x; : : trace_kfree(_RET_IP_, x); : : if (unlikely(ZERO_OR_NULL_PTR(x))) 75.08 : ffff800010253890: cmp x0, #0x10 0.00 : ffff800010253894: b.ls ffff800010253a9c // b.plast : { 0.00 : ffff800010253898: stp x29, x30, [sp, #-64]! 0.00 : ffff80001025389c: mov x29, sp 0.00 : ffff8000102538a0: stp x21, x22, [sp, #32] : virt_to_head_page(): : } : #endif : : static inline struct page *virt_to_head_page(const void *x) : { : struct page *page = virt_to_page(x); 0.00 : ffff8000102538a4: mov x21, #0x1000000000000 // #281474976710656 0.00 : ffff8000102538a8: add x21, x0, x21 : kfree(): 0.00 : ffff8000102538ac: str x19, [sp, #16] 0.00 : ffff8000102538b0: mov x19, x0 : virt_to_head_page(): 0.00 : ffff8000102538b4: mov x0, #0xffffffffffe00000 // #-2097152 0.00 : ffff8000102538b8: lsr x21, x21, #12 0.00 : ffff8000102538bc: movk x0, #0xfdff, lsl #32 0.00 : ffff8000102538c0: mov x22, x30 0.00 : ffff8000102538c4: add x21, x0, x21, lsl #6 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000102538c8: ldr x0, [x21, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff8000102538cc: sub x1, x0, #0x1 0.00 : ffff8000102538d0: tst x0, #0x1 0.00 : ffff8000102538d4: csel x21, x1, x21, ne // ne = any : __read_once_size(): 0.00 : ffff8000102538d8: ldr x1, [x21, #8] : compound_head(): 0.00 : ffff8000102538dc: sub x0, x1, #0x1 0.00 : ffff8000102538e0: tst x1, #0x1 0.00 : ffff8000102538e4: csel x0, x0, x21, ne // ne = any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000102538e8: ldr x0, [x0] : kfree(): : return; : : page = virt_to_head_page(x); : if (unlikely(!PageSlab(page))) { 0.00 : ffff8000102538ec: tst w0, #0x200 0.00 : ffff8000102538f0: b.eq ffff800010253aa0 // b.none 0.00 : ffff8000102538f4: str x20, [x29, #24] : mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE, : -(1 << order)); : __free_pages(page, order); : return; : } : slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_); 0.00 : ffff8000102538f8: ldr x20, [x21, #24] : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102538fc: nop : set_freepointer(): : unsigned long freeptr_addr = (unsigned long)object + s->offset; 0.00 : ffff800010253900: ldr w0, [x20, #32] : *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr); 0.00 : ffff800010253904: str xzr, [x19, x0] : slab_free(): : if (slab_free_freelist_hook(s, &head, &tail)) 0.00 : ffff800010253908: cbz x19, ffff800010253a54 0.00 : ffff80001025390c: str x23, [x29, #48] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010253910: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff800010253914: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010253918: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001025391c: str w0, [x1, #16] : do_slab_free(): : tid = this_cpu_read(s->cpu_slab->tid); 0.00 : ffff800010253920: ldr x0, [x20] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010253924: mrs x2, tpidr_el1 : do_slab_free(): 0.00 : ffff800010253928: add x0, x0, #0x8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001025392c: ldr x23, [x0, x2] 0.00 : ffff800010253930: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010253934: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010253938: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001025393c: cbnz x0, ffff800010253a68 : do_slab_free(): 0.00 : ffff800010253940: bl ffff800010cd2e78 : __my_cpu_offset(): 0.00 : ffff800010253944: mrs x1, tpidr_el1 : do_slab_free(): : c = raw_cpu_ptr(s->cpu_slab); 0.00 : ffff800010253948: ldr x0, [x20] 0.00 : ffff80001025394c: add x2, x0, x1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010253950: ldr x5, [x2, #8] : do_slab_free(): : unlikely(tid != READ_ONCE(c->tid))); 0.00 : ffff800010253954: cmp x5, x23 0.00 : ffff800010253958: b.ne ffff800010253910 // b.any : if (likely(page == c->page)) { 0.00 : ffff80001025395c: ldr x2, [x2, #16] 0.00 : ffff800010253960: cmp x2, x21 0.00 : ffff800010253964: b.ne ffff800010253b10 // b.any : set_freepointer(): : unsigned long freeptr_addr = (unsigned long)object + s->offset; 0.00 : ffff800010253968: ldr w2, [x20, #32] : do_slab_free(): : set_freepointer(s, tail_obj, c->freelist); 0.00 : ffff80001025396c: ldr x4, [x0, x1] : get_current(): 0.00 : ffff800010253970: mrs x3, sp_el0 : set_freepointer(): : *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr); 0.00 : ffff800010253974: str x4, [x19, x2] : __read_once_size(): 0.00 : ffff800010253978: ldr w2, [x3, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff80001025397c: add w2, w2, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010253980: str w2, [x3, #16] : do_slab_free(): : if (unlikely(!this_cpu_cmpxchg_double( 0.00 : ffff800010253984: ldr x6, [x0, x1] : next_tid(): : return tid + TID_STEP; 0.00 : ffff800010253988: add x3, x23, #0x100 : __my_cpu_offset(): 0.00 : ffff80001025398c: mrs x0, tpidr_el1 : do_slab_free(): : if (unlikely(!this_cpu_cmpxchg_double( 0.00 : ffff800010253990: ldr x4, [x20] 0.00 : ffff800010253994: add x4, x4, x0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010253998: b ffff800010253a74 0.00 : ffff80001025399c: b ffff800010253a74 : __lse__cmpxchg_double(): : : cl); \ : \ : return x0; \ : } : : __CMPXCHG_DBL( , ) 0.00 : ffff8000102539a0: mov x0, x6 0.00 : ffff8000102539a4: mov x1, x5 0.00 : ffff8000102539a8: mov x2, x19 0.00 : ffff8000102539ac: casp x0, x1, x2, x3, [x4] 0.00 : ffff8000102539b0: eor x0, x0, x6 0.00 : ffff8000102539b4: eor x1, x1, x5 0.00 : ffff8000102539b8: orr x0, x0, x1 0.00 : ffff8000102539bc: mov x23, x0 : get_current(): 0.00 : ffff8000102539c0: mrs x0, sp_el0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102539c4: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102539c8: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102539cc: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102539d0: cbnz x1, ffff800010253a8c : do_slab_free(): 0.00 : ffff8000102539d4: bl ffff800010cd2e78 0.00 : ffff8000102539d8: cbnz x23, ffff800010253910 0.00 : ffff8000102539dc: ldr x20, [x29, #24] 0.00 : ffff8000102539e0: ldr x23, [x29, #48] : kfree(): : } 0.00 : ffff8000102539e4: ldr x19, [sp, #16] 0.00 : ffff8000102539e8: ldp x21, x22, [sp, #32] 0.00 : ffff8000102539ec: ldp x29, x30, [sp], #64 0.00 : ffff8000102539f0: ret : slab_want_init_on_free(): : } : : static inline bool slab_want_init_on_free(struct kmem_cache *c) : { : if (static_branch_unlikely(&init_on_free)) : return !(c->ctor || 0.00 : ffff8000102539f4: ldr x0, [x20, #64] 0.00 : ffff8000102539f8: cbnz x0, ffff800010253900 : (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))); 0.00 : ffff8000102539fc: ldr w0, [x20, #8] 0.00 : ffff800010253a00: and w0, w0, #0xff800 0.00 : ffff800010253a04: and w0, w0, #0xfff80fff : return !(c->ctor || 0.00 : ffff800010253a08: cbnz w0, ffff800010253900 : slab_free_freelist_hook(): : memset(object, 0, s->object_size); 0.00 : ffff800010253a0c: ldr w2, [x20, #28] 0.00 : ffff800010253a10: mov w1, #0x0 // #0 0.00 : ffff800010253a14: mov x0, x19 0.00 : ffff800010253a18: bl ffff800010cb80c0 <__memset> : rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad 0.00 : ffff800010253a1c: ldr w0, [x20, #8] : : 0; 0.00 : ffff800010253a20: mov w3, #0x0 // #0 0.00 : ffff800010253a24: tbz w0, #10, ffff800010253a2c : rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad 0.00 : ffff800010253a28: ldr w3, [x20, #80] : memset((char *)object + s->inuse, 0, 0.00 : ffff800010253a2c: ldr w0, [x20, #72] 0.00 : ffff800010253a30: mov w1, #0x0 // #0 : s->size - s->inuse - rsize); 0.00 : ffff800010253a34: ldr w2, [x20, #24] 0.00 : ffff800010253a38: sub w2, w2, w0 : memset((char *)object + s->inuse, 0, 0.00 : ffff800010253a3c: add x0, x19, w0, uxtw 0.00 : ffff800010253a40: sub w2, w2, w3 0.00 : ffff800010253a44: bl ffff800010cb80c0 <__memset> : set_freepointer(): : unsigned long freeptr_addr = (unsigned long)object + s->offset; 0.00 : ffff800010253a48: ldr w0, [x20, #32] : *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr); 0.00 : ffff800010253a4c: str xzr, [x19, x0] : slab_free(): : if (slab_free_freelist_hook(s, &head, &tail)) 0.00 : ffff800010253a50: cbnz x19, ffff80001025390c 0.00 : ffff800010253a54: ldr x20, [x29, #24] : kfree(): : } 0.00 : ffff800010253a58: ldr x19, [sp, #16] 0.00 : ffff800010253a5c: ldp x21, x22, [sp, #32] 0.00 : ffff800010253a60: ldp x29, x30, [sp], #64 0.00 : ffff800010253a64: ret : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010253a68: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010253a6c: cbz x0, ffff800010253940 0.00 : ffff800010253a70: b ffff800010253944 : __ll_sc__cmpxchg_double(): : : cl); \ : \ : return ret; \ : } : : __CMPXCHG_DBL( , , , ) 0.00 : ffff800010253a74: b ffff800010255c0c : get_current(): 0.00 : ffff800010253a78: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff800010253a7c: ldr x1, [x0, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010253a80: sub x1, x1, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010253a84: str w1, [x0, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff800010253a88: cbz x1, ffff8000102539d4 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010253a8c: ldr x0, [x0, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010253a90: cbnz x0, ffff8000102539d8 : do_slab_free(): : if (unlikely(!this_cpu_cmpxchg_double( 0.00 : ffff800010253a94: bl ffff800010cd2e78 0.00 : ffff800010253a98: b ffff8000102539d8 24.92 : ffff800010253a9c: ret : test_bit(): 0.00 : ffff800010253aa0: ldr x0, [x21] : compound_order(): : } : : static inline unsigned int compound_order(struct page *page) : { : if (!PageHead(page)) : return 0; 0.00 : ffff800010253aa4: mov w19, #0x0 // #0 : if (!PageHead(page)) 0.00 : ffff800010253aa8: tst w0, #0x10000 0.00 : ffff800010253aac: b.ne ffff800010253b08 // b.any : test_bit(): 0.00 : ffff800010253ab0: ldr x0, [x21] : PageCompound(): : return READ_ONCE(page->compound_head) & 1; : } : : static __always_inline int PageCompound(struct page *page) : { : return test_bit(PG_head, &page->flags) || PageTail(page); 0.00 : ffff800010253ab4: tst w0, #0x10000 0.00 : ffff800010253ab8: b.ne ffff800010253ad0 // b.any : __read_once_size(): 0.00 : ffff800010253abc: ldr x0, [x21, #8] : PageCompound(): 0.00 : ffff800010253ac0: tbnz w0, #0, ffff800010253ad0 0.00 : ffff800010253ac4: str x20, [x29, #24] 0.00 : ffff800010253ac8: str x23, [x29, #48] : kfree(): : BUG_ON(!PageCompound(page)); 0.00 : ffff800010253acc: brk #0x800 : page_to_nid(): : #else : static inline int page_to_nid(const struct page *page) : { : struct page *p = (struct page *)page; : : return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK; 0.00 : ffff800010253ad0: ldr x3, [x21] : page_pgdat(): : return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; : } : : static inline pg_data_t *page_pgdat(const struct page *page) : { : return NODE_DATA(page_to_nid(page)); 0.00 : ffff800010253ad4: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010253ad8: add x0, x0, #0x128 : kfree(): : -(1 << order)); 0.00 : ffff800010253adc: mov w2, #0xffffffff // #-1 0.00 : ffff800010253ae0: lsl w2, w2, w19 : mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE, 0.00 : ffff800010253ae4: mov w1, #0x6 // #6 : page_pgdat(): 0.00 : ffff800010253ae8: lsr x3, x3, #62 : kfree(): 0.00 : ffff800010253aec: sxtw x2, w2 0.00 : ffff800010253af0: ldr x0, [x0, x3, lsl #3] 0.00 : ffff800010253af4: bl ffff8000101f5ef8 : __free_pages(page, order); 0.00 : ffff800010253af8: mov w1, w19 0.00 : ffff800010253afc: mov x0, x21 0.00 : ffff800010253b00: bl ffff800010227d40 <__free_pages> : return; 0.00 : ffff800010253b04: b ffff800010253a58 : compound_order(): : return page[1].compound_order; 0.00 : ffff800010253b08: ldrb w19, [x21, #81] 0.00 : ffff800010253b0c: b ffff800010253ab0 : do_slab_free(): : __slab_free(s, page, head, tail_obj, cnt, addr); 0.00 : ffff800010253b10: mov x0, x20 0.00 : ffff800010253b14: mov x5, x22 0.00 : ffff800010253b18: mov w4, #0x1 // #1 0.00 : ffff800010253b1c: mov x3, x19 0.00 : ffff800010253b20: mov x2, x19 0.00 : ffff800010253b24: mov x1, x21 0.00 : ffff800010253b28: bl ffff800010252740 <__slab_free> 0.00 : ffff800010253b2c: ldr x20, [x29, #24] 0.00 : ffff800010253b30: ldr x23, [x29, #48] 0.00 : ffff800010253b34: b ffff800010253a58 Percent | Source code & Disassembly of vmlinux for cycles (3 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101b3e98 <__acct_update_integrals>: : __acct_update_integrals(): : static void __acct_update_integrals(struct task_struct *tsk, : u64 utime, u64 stime) : { : u64 time, delta; : : if (!likely(tsk->mm)) 32.99 : ffff8000101b3e98: ldr x4, [x0, #952] 0.00 : ffff8000101b3e9c: cbz x4, ffff8000101b3f10 <__acct_update_integrals+0x78> : return; : : time = stime + utime; : delta = time - tsk->acct_timexpd; 0.00 : ffff8000101b3ea0: ldr x3, [x0, #1968] : time = stime + utime; 0.00 : ffff8000101b3ea4: add x1, x2, x1 : : if (delta < TICK_NSEC) 0.00 : ffff8000101b3ea8: mov x2, #0x8ff // #2303 : delta = time - tsk->acct_timexpd; 0.00 : ffff8000101b3eac: sub x3, x1, x3 : if (delta < TICK_NSEC) 0.00 : ffff8000101b3eb0: movk x2, #0x3d, lsl #16 0.00 : ffff8000101b3eb4: cmp x3, x2 0.00 : ffff8000101b3eb8: b.ls ffff8000101b3f10 <__acct_update_integrals+0x78> // b.plast : return; : : tsk->acct_timexpd = time; 0.00 : ffff8000101b3ebc: str x1, [x0, #1968] : /* : * Divide by 1024 to avoid overflow, and to avoid division. : * The final unit reported to userspace is Mbyte-usecs, : * the rest of the math is done in xacct_add_tsk. : */ : tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm) >> 10; 0.00 : ffff8000101b3ec0: ldr x7, [x0, #1952] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101b3ec4: ldr x1, [x4, #696] 0.00 : ffff8000101b3ec8: ldr x5, [x4, #704] : get_mm_counter(): : * But it's never be expected number for users. : */ : if (val < 0) : val = 0; : #endif : return (unsigned long)val; 0.00 : ffff8000101b3ecc: cmp x1, #0x0 : __read_once_size(): 0.00 : ffff8000101b3ed0: ldr x2, [x4, #720] : get_mm_counter(): 0.00 : ffff8000101b3ed4: csel x1, x1, xzr, ge // ge = tcont 0.00 : ffff8000101b3ed8: cmp x5, #0x0 0.00 : ffff8000101b3edc: csel x5, x5, xzr, ge // ge = tcont 0.00 : ffff8000101b3ee0: cmp x2, #0x0 : get_mm_rss(): : return mm_counter_file(page); : } : : static inline unsigned long get_mm_rss(struct mm_struct *mm) : { : return get_mm_counter(mm, MM_FILEPAGES) + 0.00 : ffff8000101b3ee4: add x1, x1, x5 : get_mm_counter(): : return (unsigned long)val; 0.00 : ffff8000101b3ee8: csel x2, x2, xzr, ge // ge = tcont : get_mm_rss(): : get_mm_counter(mm, MM_ANONPAGES) + 0.00 : ffff8000101b3eec: add x1, x1, x2 : __acct_update_integrals(): : tsk->acct_vm_mem1 += delta * tsk->mm->total_vm >> 10; 0.00 : ffff8000101b3ef0: ldr x6, [x0, #1960] : tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm) >> 10; 0.00 : ffff8000101b3ef4: mul x1, x1, x3 0.00 : ffff8000101b3ef8: add x1, x7, x1, lsr #10 0.00 : ffff8000101b3efc: str x1, [x0, #1952] : tsk->acct_vm_mem1 += delta * tsk->mm->total_vm >> 10; 0.00 : ffff8000101b3f00: ldr x1, [x4, #176] 0.00 : ffff8000101b3f04: mul x3, x3, x1 0.00 : ffff8000101b3f08: add x3, x6, x3, lsr #10 0.00 : ffff8000101b3f0c: str x3, [x0, #1960] : } 67.01 : ffff8000101b3f10: ret Percent | Source code & Disassembly of vmlinux for cycles (7 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010196d80 : : cgroup_rstat_updated(): : * @cgrp's rstat_cpu on @cpu was updated. Put it on the parent's matching : * rstat_cpu->updated_children list. See the comment on top of : * cgroup_rstat_cpu definition for details. : */ : void cgroup_rstat_updated(struct cgroup *cgrp, int cpu) : { 0.00 : ffff800010196d80: stp x29, x30, [sp, #-48]! : raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu); 0.00 : ffff800010196d84: adrp x2, ffff800011525000 0.00 : ffff800010196d88: add x2, x2, #0xfe8 : { 0.00 : ffff800010196d8c: mov x29, sp : cgroup_parent(): : : static inline struct cgroup *cgroup_parent(struct cgroup *cgrp) : { : struct cgroup_subsys_state *parent_css = cgrp->self.parent; : : if (parent_css) 0.00 : ffff800010196d90: ldr x3, [x0, #232] 0.00 : ffff800010196d94: cbz x3, ffff800010196dc8 0.00 : ffff800010196d98: stp x19, x20, [x29, #16] : cgroup_rstat_updated(): : raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu); 0.00 : ffff800010196d9c: sxtw x19, w1 0.00 : ffff800010196da0: adrp x20, ffff800011909000 0.00 : ffff800010196da4: add x20, x20, #0x928 0.00 : ffff800010196da8: ldr x4, [x20, x19, lsl #3] : : /* : * Paired with the one in cgroup_rstat_cpu_pop_updated(). Either we : * see NULL updated_next or they see our updated stat. : */ : smp_mb(); 14.59 : ffff800010196dac: dmb ish : /* : * Because @parent's updated_children is terminated with @parent : * instead of NULL, we can tell whether @cgrp is on the list by : * testing the next pointer for NULL. : */ : if (cgroup_rstat_cpu(cgrp, cpu)->updated_next) 57.15 : ffff800010196db0: ldr x3, [x20, x19, lsl #3] : cgroup_rstat_cpu(): : return per_cpu_ptr(cgrp->rstat_cpu, cpu); 0.00 : ffff800010196db4: ldr x1, [x0, #672] : cgroup_rstat_updated(): : if (cgroup_rstat_cpu(cgrp, cpu)->updated_next) 0.00 : ffff800010196db8: add x1, x3, x1 0.00 : ffff800010196dbc: ldr x1, [x1, #56] 0.00 : ffff800010196dc0: cbz x1, ffff800010196dd0 28.26 : ffff800010196dc4: ldp x19, x20, [x29, #16] : rstatc->updated_next = prstatc->updated_children; : prstatc->updated_children = cgrp; : } : : raw_spin_unlock_irqrestore(cpu_lock, flags); : } 0.00 : ffff800010196dc8: ldp x29, x30, [sp], #48 0.00 : ffff800010196dcc: ret 0.00 : ffff800010196dd0: stp x21, x22, [x29, #32] 0.00 : ffff800010196dd4: mov x21, x0 : raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu); 0.00 : ffff800010196dd8: add x22, x4, x2 : raw_spin_lock_irqsave(cpu_lock, flags); 0.00 : ffff800010196ddc: mov x0, x22 0.00 : ffff800010196de0: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : cgroup_parent(): : struct cgroup_subsys_state *parent_css = cgrp->self.parent; 0.00 : ffff800010196de4: ldr x5, [x21, #232] : cgroup_rstat_updated(): 0.00 : ffff800010196de8: mov x1, x0 : cgroup_parent(): : if (parent_css) 0.00 : ffff800010196dec: cbz x5, ffff800010196e58 : cgroup_rstat_cpu(): : return per_cpu_ptr(cgrp->rstat_cpu, cpu); 0.00 : ffff800010196df0: ldr x0, [x20, x19, lsl #3] 0.00 : ffff800010196df4: ldr x3, [x21, #672] 0.00 : ffff800010196df8: add x3, x0, x3 0.00 : ffff800010196dfc: ldr x4, [x5, #672] 0.00 : ffff800010196e00: add x4, x4, x0 : cgroup_rstat_updated(): : if (rstatc->updated_next) 0.00 : ffff800010196e04: ldr x0, [x3, #56] 0.00 : ffff800010196e08: cbnz x0, ffff800010196e58 : rstatc->updated_next = prstatc->updated_children; 0.00 : ffff800010196e0c: ldr x0, [x4, #48] 0.00 : ffff800010196e10: str x0, [x3, #56] : prstatc->updated_children = cgrp; 0.00 : ffff800010196e14: str x21, [x4, #48] : cgroup_parent(): : struct cgroup_subsys_state *parent_css = cgrp->self.parent; 0.00 : ffff800010196e18: ldr x4, [x5, #232] : if (parent_css) 0.00 : ffff800010196e1c: cbz x4, ffff800010196e58 : cgroup_rstat_cpu(): : return per_cpu_ptr(cgrp->rstat_cpu, cpu); 0.00 : ffff800010196e20: ldr x6, [x20, x19, lsl #3] 0.00 : ffff800010196e24: ldr x2, [x5, #672] 0.00 : ffff800010196e28: add x2, x2, x6 0.00 : ffff800010196e2c: ldr x3, [x4, #672] 0.00 : ffff800010196e30: add x3, x3, x6 : cgroup_rstat_updated(): : if (rstatc->updated_next) 0.00 : ffff800010196e34: ldr x6, [x2, #56] 0.00 : ffff800010196e38: cbnz x6, ffff800010196e58 : rstatc->updated_next = prstatc->updated_children; 0.00 : ffff800010196e3c: ldr x6, [x3, #48] 0.00 : ffff800010196e40: str x6, [x2, #56] : prstatc->updated_children = cgrp; 0.00 : ffff800010196e44: str x5, [x3, #48] 0.00 : ffff800010196e48: mov x5, x4 : cgroup_parent(): : struct cgroup_subsys_state *parent_css = cgrp->self.parent; 0.00 : ffff800010196e4c: ldr x3, [x4, #232] : if (parent_css) 0.00 : ffff800010196e50: mov x4, x3 0.00 : ffff800010196e54: cbnz x3, ffff800010196e20 : cgroup_rstat_updated(): : raw_spin_unlock_irqrestore(cpu_lock, flags); 0.00 : ffff800010196e58: mov x0, x22 0.00 : ffff800010196e5c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff800010196e60: ldp x19, x20, [x29, #16] 0.00 : ffff800010196e64: ldp x21, x22, [x29, #32] : } 0.00 : ffff800010196e68: ldp x29, x30, [sp], #48 0.00 : ffff800010196e6c: ret Percent | Source code & Disassembly of vmlinux for cycles (3 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010162b90 : : rcu_segcblist_ready_cbs(): : * Does the specified rcu_segcblist structure contain callbacks that : * are ready to be invoked? : */ : bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp) : { : return rcu_segcblist_is_enabled(rsclp) && 66.31 : ffff800010162b90: ldrb w1, [x0, #80] 0.00 : ffff800010162b94: cbz w1, ffff800010162ba8 33.69 : ffff800010162b98: ldr x1, [x0, #8] 0.00 : ffff800010162b9c: cmp x1, x0 0.00 : ffff800010162ba0: cset w0, ne // ne = any : &rsclp->head != rsclp->tails[RCU_DONE_TAIL]; : } 0.00 : ffff800010162ba4: ret 0.00 : ffff800010162ba8: mov w0, #0x0 // #0 0.00 : ffff800010162bac: ret Percent | Source code & Disassembly of vmlinux for cycles (8 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010120a38 : : put_prev_task_fair(): : : /* : * Account for a descheduled task: : */ : static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) : { 25.23 : ffff800010120a38: stp x29, x30, [sp, #-32]! 0.00 : ffff800010120a3c: mov x29, sp 24.03 : ffff800010120a40: str x19, [sp, #16] : struct sched_entity *se = &prev->se; : struct cfs_rq *cfs_rq; : : for_each_sched_entity(se) { 0.00 : ffff800010120a44: adds x19, x1, #0xc0 0.00 : ffff800010120a48: b.eq ffff800010120a64 // b.none 24.59 : ffff800010120a4c: nop : cfs_rq = cfs_rq_of(se); : put_prev_entity(cfs_rq, se); 12.94 : ffff800010120a50: ldr x0, [x19, #128] 0.00 : ffff800010120a54: mov x1, x19 0.00 : ffff800010120a58: bl ffff8000101209d0 : for_each_sched_entity(se) { 0.00 : ffff800010120a5c: ldr x19, [x19, #120] 0.00 : ffff800010120a60: cbnz x19, ffff800010120a50 : } : } 0.00 : ffff800010120a64: ldr x19, [sp, #16] 13.21 : ffff800010120a68: ldp x29, x30, [sp], #32 0.00 : ffff800010120a6c: ret Percent | Source code & Disassembly of vmlinux for cycles (3 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b790 : : tick_sched_timer(): : /* : * We rearm the timer until we get disabled by the idle code. : * Called with interrupts disabled. : */ : static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) : { 0.00 : ffff80001017b790: stp x29, x30, [sp, #-48]! : get_irq_regs(): : */ : DECLARE_PER_CPU(struct pt_regs *, __irq_regs); : : static inline struct pt_regs *get_irq_regs(void) : { : return __this_cpu_read(__irq_regs); 0.00 : ffff80001017b794: adrp x1, ffff800011528000 0.00 : ffff80001017b798: add x1, x1, #0x780 : tick_sched_timer(): 0.00 : ffff80001017b79c: mov x29, sp 0.00 : ffff80001017b7a0: stp x19, x20, [sp, #16] 0.00 : ffff80001017b7a4: mov x19, x0 0.00 : ffff80001017b7a8: str x21, [sp, #32] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001017b7ac: mrs x0, tpidr_el1 : get_irq_regs(): 0.00 : ffff80001017b7b0: ldr x21, [x1, x0] : tick_sched_timer(): : struct tick_sched *ts = : container_of(timer, struct tick_sched, sched_timer); : struct pt_regs *regs = get_irq_regs(); : ktime_t now = ktime_get(); 0.00 : ffff80001017b7b4: bl ffff80001016c938 0.00 : ffff80001017b7b8: mov x20, x0 : : tick_sched_do_timer(ts, now); 0.00 : ffff80001017b7bc: mov x1, x0 0.00 : ffff80001017b7c0: mov x0, x19 0.00 : ffff80001017b7c4: bl ffff80001017b710 : : /* : * Do not call, when we are not in irq context and have : * no valid regs pointer : */ : if (regs) 0.00 : ffff80001017b7c8: cbz x21, ffff80001017b808 : tick_sched_handle(ts, regs); 35.58 : ffff80001017b7cc: mov x0, x19 0.00 : ffff80001017b7d0: add x1, x21, #0x108 0.00 : ffff80001017b7d4: bl ffff80001017b670 : else : ts->next_tick = 0; : : /* No need to reprogram if we are in idle or full dynticks mode */ : if (unlikely(ts->tick_stopped)) 0.00 : ffff80001017b7d8: ldrb w0, [x19, #76] 0.00 : ffff80001017b7dc: tbnz w0, #1, ffff80001017b814 : return HRTIMER_NORESTART; : : hrtimer_forward(timer, now, tick_period); 0.00 : ffff80001017b7e0: adrp x2, ffff800011b22000 0.00 : ffff80001017b7e4: mov x1, x20 0.00 : ffff80001017b7e8: mov x0, x19 0.00 : ffff80001017b7ec: ldr x2, [x2, #968] 0.00 : ffff80001017b7f0: bl ffff80001016acf8 : : return HRTIMER_RESTART; : } 35.08 : ffff80001017b7f4: ldr x21, [sp, #32] : return HRTIMER_RESTART; 0.00 : ffff80001017b7f8: mov w0, #0x1 // #1 : } 29.34 : ffff80001017b7fc: ldp x19, x20, [sp, #16] 0.00 : ffff80001017b800: ldp x29, x30, [sp], #48 0.00 : ffff80001017b804: ret : if (unlikely(ts->tick_stopped)) 0.00 : ffff80001017b808: ldrb w0, [x19, #76] : ts->next_tick = 0; 0.00 : ffff80001017b80c: str xzr, [x19, #88] : if (unlikely(ts->tick_stopped)) 0.00 : ffff80001017b810: tbz w0, #1, ffff80001017b7e0 : return HRTIMER_NORESTART; 0.00 : ffff80001017b814: mov w0, #0x0 // #0 : } 0.00 : ffff80001017b818: ldr x21, [sp, #32] 0.00 : ffff80001017b81c: ldp x19, x20, [sp, #16] 0.00 : ffff80001017b820: ldp x29, x30, [sp], #48 0.00 : ffff80001017b824: ret Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012fcd0 : : prepare_to_wait_event(): : INIT_LIST_HEAD(&wq_entry->entry); : } : EXPORT_SYMBOL(init_wait_entry); : : long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) : { 0.00 : ffff80001012fcd0: stp x29, x30, [sp, #-64]! 0.00 : ffff80001012fcd4: mov x29, sp 0.00 : ffff80001012fcd8: stp x19, x20, [sp, #16] 0.00 : ffff80001012fcdc: adrp x20, ffff800011909000 0.00 : ffff80001012fce0: stp x21, x22, [sp, #32] 0.00 : ffff80001012fce4: add x3, x20, #0x908 0.00 : ffff80001012fce8: mov x21, x1 0.00 : ffff80001012fcec: sxtw x19, w2 24.77 : ffff80001012fcf0: ldr x1, [x3] 0.00 : ffff80001012fcf4: str x1, [x29, #56] 0.00 : ffff80001012fcf8: mov x1, #0x0 // #0 0.00 : ffff80001012fcfc: mov x22, x0 : unsigned long flags; : long ret = 0; : : spin_lock_irqsave(&wq_head->lock, flags); 0.00 : ffff80001012fd00: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : signal_pending_state(): : return signal_pending(p) && __fatal_signal_pending(p); : } : : static inline int signal_pending_state(long state, struct task_struct *p) : { : if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL))) 0.00 : ffff80001012fd04: mov w1, #0x101 // #257 0.00 : ffff80001012fd08: tst w19, w1 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 23.77 : ffff80001012fd0c: mrs x1, sp_el0 : signal_pending_state(): 0.00 : ffff80001012fd10: b.eq ffff80001012fd1c // b.none : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001012fd14: ldr x2, [x1] : signal_pending_state(): : return 0; : if (!signal_pending(p)) 0.00 : ffff80001012fd18: tbnz w2, #0, ffff80001012fda4 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001012fd1c: ldr x1, [x21, #24] : prepare_to_wait_event(): : * we fail. : */ : list_del_init(&wq_entry->entry); : ret = -ERESTARTSYS; : } else { : if (list_empty(&wq_entry->entry)) { 0.00 : ffff80001012fd20: add x2, x21, #0x18 0.00 : ffff80001012fd24: cmp x2, x1 0.00 : ffff80001012fd28: b.eq ffff80001012fd70 // b.none : get_current(): 0.00 : ffff80001012fd2c: mrs x1, sp_el0 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012fd30: str x19, [x1, #24] : prepare_to_wait_event(): : if (wq_entry->flags & WQ_FLAG_EXCLUSIVE) : __add_wait_queue_entry_tail(wq_head, wq_entry); : else : __add_wait_queue(wq_head, wq_entry); : } : set_current_state(state); 0.00 : ffff80001012fd34: dmb ish : long ret = 0; 26.05 : ffff80001012fd38: mov x19, #0x0 // #0 : spin_unlock_irqrestore(): : raw_spin_unlock_irq(&lock->rlock); : } : : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) : { : raw_spin_unlock_irqrestore(&lock->rlock, flags); 0.00 : ffff80001012fd3c: mov x1, x0 : prepare_to_wait_event(): : } : spin_unlock_irqrestore(&wq_head->lock, flags); : : return ret; : } 0.00 : ffff80001012fd40: add x20, x20, #0x908 : spin_unlock_irqrestore(): 0.00 : ffff80001012fd44: mov x0, x22 0.00 : ffff80001012fd48: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : prepare_to_wait_event(): 0.00 : ffff80001012fd4c: mov x0, x19 0.00 : ffff80001012fd50: ldr x2, [x29, #56] 0.00 : ffff80001012fd54: ldr x1, [x20] 0.00 : ffff80001012fd58: eor x1, x2, x1 0.00 : ffff80001012fd5c: cbnz x1, ffff80001012fdd4 0.00 : ffff80001012fd60: ldp x19, x20, [sp, #16] 0.00 : ffff80001012fd64: ldp x21, x22, [sp, #32] 0.00 : ffff80001012fd68: ldp x29, x30, [sp], #64 0.00 : ffff80001012fd6c: ret : if (wq_entry->flags & WQ_FLAG_EXCLUSIVE) 0.00 : ffff80001012fd70: ldr w3, [x21] 0.00 : ffff80001012fd74: add x1, x22, #0x8 0.00 : ffff80001012fd78: tbz w3, #0, ffff80001012fd90 : list_add_tail(): : * Insert a new entry before the specified head. : * This is useful for implementing queues. : */ : static inline void list_add_tail(struct list_head *new, struct list_head *head) : { : __list_add(new, head->prev, head); 0.00 : ffff80001012fd7c: ldr x3, [x22, #16] : __list_add(): : next->prev = new; 0.00 : ffff80001012fd80: str x2, [x22, #16] : new->prev = prev; 0.00 : ffff80001012fd84: stp x1, x3, [x21, #24] : __write_once_size(): 0.00 : ffff80001012fd88: str x2, [x3] 0.00 : ffff80001012fd8c: b ffff80001012fd2c : list_add(): : __list_add(new, head, head->next); 0.00 : ffff80001012fd90: ldr x3, [x22, #8] : __list_add(): : next->prev = new; 0.00 : ffff80001012fd94: str x2, [x3, #8] : new->prev = prev; 25.40 : ffff80001012fd98: stp x3, x1, [x21, #24] : __write_once_size(): 0.00 : ffff80001012fd9c: str x2, [x22, #8] 0.00 : ffff80001012fda0: b ffff80001012fd2c : signal_pending_state(): : return 0; : : return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); 0.00 : ffff80001012fda4: tbnz w19, #0, ffff80001012fdb4 : sigismember(): : : static inline int sigismember(sigset_t *set, int _sig) : { : unsigned long sig = _sig - 1; : if (_NSIG_WORDS == 1) : return 1 & (set->sig[0] >> sig); 0.00 : ffff80001012fda8: ldr x1, [x1, #1696] : signal_pending_state(): 0.00 : ffff80001012fdac: tst w1, #0x100 0.00 : ffff80001012fdb0: b.eq ffff80001012fd1c // b.none : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff80001012fdb4: ldp x3, x2, [x21, #24] : __list_del(): : next->prev = prev; 0.00 : ffff80001012fdb8: str x2, [x3, #8] : prepare_to_wait_event(): : list_del_init(&wq_entry->entry); 0.00 : ffff80001012fdbc: add x1, x21, #0x18 : ret = -ERESTARTSYS; 0.00 : ffff80001012fdc0: mov x19, #0xfffffffffffffe00 // #-512 : __write_once_size(): 0.00 : ffff80001012fdc4: str x3, [x2] 0.00 : ffff80001012fdc8: str x1, [x21, #24] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff80001012fdcc: str x1, [x21, #32] : prepare_to_wait_event(): 0.00 : ffff80001012fdd0: b ffff80001012fd3c : } 0.00 : ffff80001012fdd4: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (3 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011c5c8 : : update_load_sub(): : lw->inv_weight = 0; : } : : static inline void update_load_sub(struct load_weight *lw, unsigned long dec) : { : lw->weight -= dec; 0.00 : ffff80001011c5c8: ldr x3, [x1] 0.00 : ffff80001011c5cc: ldr x2, [x0] : lw->inv_weight = 0; 0.00 : ffff80001011c5d0: str wzr, [x0, #8] : lw->weight -= dec; 0.00 : ffff80001011c5d4: sub x2, x2, x3 0.00 : ffff80001011c5d8: str x2, [x0] : account_entity_dequeue(): : static void : account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) : { : update_load_sub(&cfs_rq->load, se->load.weight); : #ifdef CONFIG_SMP : if (entity_is_task(se)) { 0.00 : ffff80001011c5dc: ldr x2, [x1, #136] 0.00 : ffff80001011c5e0: cbnz x2, ffff80001011c64c : account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 0.00 : ffff80001011c5e4: ldr x3, [x0, #304] : cpu_to_node(): : #endif : : #ifndef cpu_to_node : static inline int cpu_to_node(int cpu) : { : return per_cpu(numa_node, cpu); 0.00 : ffff80001011c5e8: adrp x5, ffff800011909000 : account_numa_dequeue(): : rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 0.00 : ffff80001011c5ec: ldr w7, [x1, #2000] : cpu_to_node(): 0.00 : ffff80001011c5f0: add x5, x5, #0x928 0.00 : ffff80001011c5f4: adrp x2, ffff800011526000 0.00 : ffff80001011c5f8: add x2, x2, #0xd58 : account_numa_dequeue(): 0.00 : ffff80001011c5fc: ldr w4, [x3, #8] 0.00 : ffff80001011c600: cmn w7, #0x1 0.00 : ffff80001011c604: cset w7, ne // ne = any : account_entity_dequeue(): : list_del_init(&se->group_node); 0.00 : ffff80001011c608: add x6, x1, #0x30 : account_numa_dequeue(): : rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 0.00 : ffff80001011c60c: sub w4, w4, w7 0.00 : ffff80001011c610: str w4, [x3, #8] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011c614: ldur w8, [x1, #-124] : account_numa_dequeue(): : rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 0.00 : ffff80001011c618: ldr w7, [x1, #2000] 0.00 : ffff80001011c61c: ldr w4, [x3, #12] : cpu_to_node(): 0.00 : ffff80001011c620: ldr x5, [x5, w8, sxtw #3] : account_numa_dequeue(): 34.34 : ffff80001011c624: ldr w2, [x5, x2] 0.00 : ffff80001011c628: cmp w2, w7 0.00 : ffff80001011c62c: cset w2, eq // eq = none 0.00 : ffff80001011c630: sub w2, w4, w2 34.52 : ffff80001011c634: str w2, [x3, #12] : __list_del_entry(): : static inline void __list_del_entry(struct list_head *entry) : { : if (!__list_del_entry_valid(entry)) : return; : : __list_del(entry->prev, entry->next); 0.00 : ffff80001011c638: ldp x3, x2, [x1, #48] : __list_del(): : next->prev = prev; 0.00 : ffff80001011c63c: str x2, [x3, #8] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001011c640: str x3, [x2] 0.00 : ffff80001011c644: str x6, [x1, #48] : INIT_LIST_HEAD(): : list->prev = list; 0.00 : ffff80001011c648: str x6, [x1, #56] : account_entity_dequeue(): : } : #endif : cfs_rq->nr_running--; 0.00 : ffff80001011c64c: ldr w1, [x0, #24] 0.00 : ffff80001011c650: sub w1, w1, #0x1 31.13 : ffff80001011c654: str w1, [x0, #24] : } 0.00 : ffff80001011c658: ret Percent | Source code & Disassembly of vmlinux for cycles (3 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101f3c20 : : first_online_pgdat(): : #include : #include : #include : : struct pglist_data *first_online_pgdat(void) : { 31.37 : ffff8000101f3c20: stp x29, x30, [sp, #-16]! : __first_node(): : > MAX_NUMNODES, then the silly min_ts could be dropped. */ : : #define first_node(src) __first_node(&(src)) : static inline int __first_node(const nodemask_t *srcp) : { : return min_t(int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES)); 0.00 : ffff8000101f3c24: mov x2, #0x0 // #0 0.00 : ffff8000101f3c28: mov x1, #0x4 // #4 0.00 : ffff8000101f3c2c: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> : first_online_pgdat(): 0.00 : ffff8000101f3c30: mov x29, sp : __first_node(): 0.00 : ffff8000101f3c34: add x0, x0, #0x428 0.00 : ffff8000101f3c38: bl ffff80001048daa8 0.00 : ffff8000101f3c3c: cmp w0, #0x4 0.00 : ffff8000101f3c40: mov w2, #0x4 // #4 : first_online_pgdat(): : return NODE_DATA(first_online_node); 0.00 : ffff8000101f3c44: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : __first_node(): 0.00 : ffff8000101f3c48: csel w0, w0, w2, le : first_online_pgdat(): 0.00 : ffff8000101f3c4c: add x1, x1, #0x128 : } 0.00 : ffff8000101f3c50: ldp x29, x30, [sp], #16 68.63 : ffff8000101f3c54: ldr x0, [x1, w0, sxtw #3] 0.00 : ffff8000101f3c58: ret Percent | Source code & Disassembly of vmlinux for cycles (3 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104850b8 <__bitmap_and>: : __bitmap_and(): : : int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, : const unsigned long *bitmap2, unsigned int bits) : { : unsigned int k; : unsigned int lim = bits/BITS_PER_LONG; 0.00 : ffff8000104850b8: lsr w8, w3, #6 : unsigned long result = 0; : : for (k = 0; k < lim; k++) 0.00 : ffff8000104850bc: cbz w8, ffff800010485120 <__bitmap_and+0x68> 0.00 : ffff8000104850c0: mov x4, #0x0 // #0 : unsigned long result = 0; 0.00 : ffff8000104850c4: mov x6, #0x0 // #0 : result |= (dst[k] = bitmap1[k] & bitmap2[k]); 34.59 : ffff8000104850c8: ldr x5, [x1, x4, lsl #3] 33.37 : ffff8000104850cc: ldr x7, [x2, x4, lsl #3] 0.00 : ffff8000104850d0: and x5, x5, x7 0.00 : ffff8000104850d4: str x5, [x0, x4, lsl #3] 0.00 : ffff8000104850d8: add x4, x4, #0x1 0.00 : ffff8000104850dc: orr x6, x6, x5 : for (k = 0; k < lim; k++) 0.00 : ffff8000104850e0: cmp w8, w4 32.04 : ffff8000104850e4: b.hi ffff8000104850c8 <__bitmap_and+0x10> // b.pmore : if (bits % BITS_PER_LONG) 0.00 : ffff8000104850e8: tst x3, #0x3f 0.00 : ffff8000104850ec: b.eq ffff800010485114 <__bitmap_and+0x5c> // b.none : result |= (dst[k] = bitmap1[k] & bitmap2[k] & : BITMAP_LAST_WORD_MASK(bits)); 0.00 : ffff8000104850f0: neg w3, w3 0.00 : ffff8000104850f4: mov x4, #0xffffffffffffffff // #-1 0.00 : ffff8000104850f8: lsr x4, x4, x3 : result |= (dst[k] = bitmap1[k] & bitmap2[k] & 0.00 : ffff8000104850fc: ldr x3, [x1, x8, lsl #3] 0.00 : ffff800010485100: ldr x1, [x2, x8, lsl #3] 0.00 : ffff800010485104: and x3, x3, x1 0.00 : ffff800010485108: and x3, x3, x4 0.00 : ffff80001048510c: str x3, [x0, x8, lsl #3] 0.00 : ffff800010485110: orr x6, x6, x3 : return result != 0; 0.00 : ffff800010485114: cmp x6, #0x0 : } 0.00 : ffff800010485118: cset w0, ne // ne = any 0.00 : ffff80001048511c: ret : unsigned long result = 0; 0.00 : ffff800010485120: mov x6, #0x0 // #0 0.00 : ffff800010485124: b ffff8000104850e8 <__bitmap_and+0x30> Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100a5400 : : pud_huge(): : } : : int pud_huge(pud_t pud) : { : #ifndef __PAGETABLE_PMD_FOLDED : return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT); 100.00 : ffff8000100a5400: cmp x0, #0x0 0.00 : ffff8000100a5404: lsr x0, x0, #1 0.00 : ffff8000100a5408: eor x0, x0, #0x1 0.00 : ffff8000100a540c: and w0, w0, #0x1 : #else : return 0; : #endif : } 0.00 : ffff8000100a5410: csel w0, w0, wzr, ne // ne = any 0.00 : ffff8000100a5414: ret Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010470618 : : part_inc_in_flight(): : static void disk_del_events(struct gendisk *disk); : static void disk_release_events(struct gendisk *disk); : : void part_inc_in_flight(struct request_queue *q, struct hd_struct *part, int rw) : { : if (queue_is_mq(q)) 25.24 : ffff800010470618: ldr x0, [x0, #48] 0.00 : ffff80001047061c: cbz x0, ffff800010470624 : return; : : part_stat_local_inc(part, in_flight[rw]); : if (part->partno) : part_stat_local_inc(&part_to_disk(part)->part0, in_flight[rw]); : } 74.76 : ffff800010470620: ret : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010470624: mrs x5, tpidr_el1 : part_inc_in_flight(): : part_stat_local_inc(part, in_flight[rw]); 0.00 : ffff800010470628: adrp x3, ffff80001151d000 0.00 : ffff80001047062c: add x0, x3, #0x18 0.00 : ffff800010470630: ldrsw x6, [x0, x5] 0.00 : ffff800010470634: adrp x4, ffff800011909000 0.00 : ffff800010470638: add x5, x4, #0x928 0.00 : ffff80001047063c: sxtw x2, w2 0.00 : ffff800010470640: add x2, x2, #0x12 0.00 : ffff800010470644: ldr x0, [x1, #840] 0.00 : ffff800010470648: ldr x5, [x5, x6, lsl #3] 0.00 : ffff80001047064c: lsl x2, x2, #3 0.00 : ffff800010470650: add x0, x0, x5 0.00 : ffff800010470654: add x0, x0, x2 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010470658: b ffff8000104706b8 0.00 : ffff80001047065c: b ffff8000104706b8 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff800010470660: mov x5, #0x1 // #1 0.00 : ffff800010470664: stadd x5, [x0] : part_inc_in_flight(): : if (part->partno) 0.00 : ffff800010470668: ldr w0, [x1, #820] 0.00 : ffff80001047066c: cbz w0, ffff800010470620 : part_to_disk(): : struct lockdep_map lockdep_map; : }; : : static inline struct gendisk *part_to_disk(struct hd_struct *part) : { : if (likely(part)) { 0.00 : ffff800010470670: cbz x1, ffff8000104706c0 : if (part->partno) : return dev_to_disk(part_to_dev(part)->parent); 0.00 : ffff800010470674: ldr x0, [x1, #104] 0.00 : ffff800010470678: sub x0, x0, #0x70 : part_inc_in_flight(): : part_stat_local_inc(&part_to_disk(part)->part0, in_flight[rw]); 0.00 : ffff80001047067c: add x3, x3, #0x18 0.00 : ffff800010470680: add x4, x4, #0x928 : __my_cpu_offset(): 0.00 : ffff800010470684: mrs x1, tpidr_el1 : part_inc_in_flight(): 0.00 : ffff800010470688: ldrsw x1, [x3, x1] 0.00 : ffff80001047068c: ldr x0, [x0, #912] 0.00 : ffff800010470690: ldr x1, [x4, x1, lsl #3] 0.00 : ffff800010470694: add x0, x0, x1 0.00 : ffff800010470698: add x2, x0, x2 : arch_static_branch_jump(): 0.00 : ffff80001047069c: b ffff8000104706b0 0.00 : ffff8000104706a0: b ffff8000104706b0 : __lse_atomic64_add(): 0.00 : ffff8000104706a4: mov x0, #0x1 // #1 0.00 : ffff8000104706a8: stadd x0, [x2] : part_inc_in_flight(): : } 0.00 : ffff8000104706ac: ret : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff8000104706b0: b ffff800010471f04 : part_inc_in_flight(): 0.00 : ffff8000104706b4: ret : __ll_sc_atomic64_add(): 0.00 : ffff8000104706b8: b ffff800010471f1c 0.00 : ffff8000104706bc: b ffff800010470668 : part_to_disk(): : else : return dev_to_disk(part_to_dev(part)); : } : return NULL; 0.00 : ffff8000104706c0: mov x0, #0x0 // #0 0.00 : ffff8000104706c4: b ffff80001047067c Percent | Source code & Disassembly of vmlinux for cycles (5 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101d4ad8 : : filemap_write_and_wait_range(): : * : * Return: error status of the address space. : */ : int filemap_write_and_wait_range(struct address_space *mapping, : loff_t lstart, loff_t lend) : { 0.00 : ffff8000101d4ad8: stp x29, x30, [sp, #-48]! 0.00 : ffff8000101d4adc: mov x29, sp 0.00 : ffff8000101d4ae0: str x19, [sp, #16] : int err = 0; : : if (mapping_needs_writeback(mapping)) { 19.92 : ffff8000101d4ae4: ldr x3, [x0, #88] 0.00 : ffff8000101d4ae8: cbz x3, ffff8000101d4b4c 0.00 : ffff8000101d4aec: stp x20, x21, [x29, #24] : err = __filemap_fdatawrite_range(mapping, lstart, lend, 0.00 : ffff8000101d4af0: mov w3, #0x1 // #1 0.00 : ffff8000101d4af4: str x22, [x29, #40] 0.00 : ffff8000101d4af8: mov x20, x0 0.00 : ffff8000101d4afc: mov x21, x1 0.00 : ffff8000101d4b00: mov x22, x2 0.00 : ffff8000101d4b04: bl ffff8000101d4960 <__filemap_fdatawrite_range> 0.00 : ffff8000101d4b08: mov w19, w0 : * Even if the above returned error, the pages may be : * written partially (e.g. -ENOSPC), so we wait for it. : * But the -EIO is special case, it may indicate the worst : * thing (e.g. bug) happened, so we avoid waiting for it. : */ : if (err != -EIO) { 0.00 : ffff8000101d4b0c: cmn w0, #0x5 0.00 : ffff8000101d4b10: b.eq ffff8000101d4b64 // b.none : filemap_fdatawait_range(): : __filemap_fdatawait_range(mapping, start_byte, end_byte); 0.00 : ffff8000101d4b14: mov x2, x22 0.00 : ffff8000101d4b18: mov x1, x21 0.00 : ffff8000101d4b1c: mov x0, x20 0.00 : ffff8000101d4b20: bl ffff8000101d0a70 <__filemap_fdatawait_range> : return filemap_check_errors(mapping); 0.00 : ffff8000101d4b24: mov x0, x20 0.00 : ffff8000101d4b28: bl ffff8000101d1bd0 0.00 : ffff8000101d4b2c: ldr x22, [x29, #40] : filemap_write_and_wait_range(): : int err2 = filemap_fdatawait_range(mapping, : lstart, lend); : if (!err) : err = err2; 0.00 : ffff8000101d4b30: cmp w19, #0x0 0.00 : ffff8000101d4b34: csel w19, w19, w0, ne // ne = any : } : } else { : err = filemap_check_errors(mapping); : } : return err; : } 0.00 : ffff8000101d4b38: mov w0, w19 0.00 : ffff8000101d4b3c: ldr x19, [sp, #16] 0.00 : ffff8000101d4b40: ldp x20, x21, [x29, #24] 0.00 : ffff8000101d4b44: ldp x29, x30, [sp], #48 0.00 : ffff8000101d4b48: ret : err = filemap_check_errors(mapping); 0.00 : ffff8000101d4b4c: bl ffff8000101d1bd0 39.99 : ffff8000101d4b50: mov w19, w0 : } 0.00 : ffff8000101d4b54: mov w0, w19 40.10 : ffff8000101d4b58: ldr x19, [sp, #16] 0.00 : ffff8000101d4b5c: ldp x29, x30, [sp], #48 0.00 : ffff8000101d4b60: ret : filemap_check_errors(mapping); 0.00 : ffff8000101d4b64: mov x0, x20 0.00 : ffff8000101d4b68: bl ffff8000101d1bd0 0.00 : ffff8000101d4b6c: ldp x20, x21, [x29, #24] 0.00 : ffff8000101d4b70: ldr x22, [x29, #40] 0.00 : ffff8000101d4b74: b ffff8000101d4b54 Percent | Source code & Disassembly of vmlinux for cycles (3 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102d88b8 : : aio_setup_rw(): : : req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ : return 0; : } : : static ssize_t aio_setup_rw(int rw, const struct iocb *iocb, 0.00 : ffff8000102d88b8: stp x29, x30, [sp, #-32]! : struct iov_iter *iter) : { : void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf; : size_t len = iocb->aio_nbytes; : : if (!vectored) { 0.00 : ffff8000102d88bc: tst w4, #0xff : static ssize_t aio_setup_rw(int rw, const struct iocb *iocb, 0.00 : ffff8000102d88c0: mov x29, sp 0.00 : ffff8000102d88c4: str x19, [sp, #16] 0.00 : ffff8000102d88c8: mov x19, x3 : if (!vectored) { 0.00 : ffff8000102d88cc: b.eq ffff8000102d8910 // b.none 0.00 : ffff8000102d88d0: and w5, w5, #0xff : ssize_t ret = import_single_range(rw, buf, len, *iovec, iter); : *iovec = NULL; : return ret; : } : #ifdef CONFIG_COMPAT : if (compat) 0.00 : ffff8000102d88d4: cbnz w5, ffff8000102d88f4 : return compat_import_iovec(rw, buf, len, UIO_FASTIOV, iovec, : iter); : #endif : return import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter); 0.00 : ffff8000102d88d8: mov x4, x3 0.00 : ffff8000102d88dc: mov x5, x6 0.00 : ffff8000102d88e0: mov w3, #0x8 // #8 0.00 : ffff8000102d88e4: bl ffff800010488220 : } 0.00 : ffff8000102d88e8: ldr x19, [sp, #16] 0.00 : ffff8000102d88ec: ldp x29, x30, [sp], #32 0.00 : ffff8000102d88f0: ret : return compat_import_iovec(rw, buf, len, UIO_FASTIOV, iovec, 0.00 : ffff8000102d88f4: mov x4, x3 0.00 : ffff8000102d88f8: mov x5, x6 0.00 : ffff8000102d88fc: mov w3, #0x8 // #8 0.00 : ffff8000102d8900: bl ffff8000104882f0 : } 0.00 : ffff8000102d8904: ldr x19, [sp, #16] 0.00 : ffff8000102d8908: ldp x29, x30, [sp], #32 0.00 : ffff8000102d890c: ret : ssize_t ret = import_single_range(rw, buf, len, *iovec, iter); 33.50 : ffff8000102d8910: ldr x3, [x3] 0.00 : ffff8000102d8914: mov x4, x6 0.00 : ffff8000102d8918: bl ffff8000104884f8 : *iovec = NULL; 0.00 : ffff8000102d891c: str xzr, [x19] : ssize_t ret = import_single_range(rw, buf, len, *iovec, iter); 0.00 : ffff8000102d8920: sxtw x0, w0 : } 66.50 : ffff8000102d8924: ldr x19, [sp, #16] 0.00 : ffff8000102d8928: ldp x29, x30, [sp], #32 0.00 : ffff8000102d892c: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015eda8 : : invoke_rcu_core(): : : /* : * Wake up this CPU's rcuc kthread to do RCU core processing. : */ : static void invoke_rcu_core(void) : { 0.00 : ffff80001015eda8: stp x29, x30, [sp, #-32]! : if (!cpu_online(smp_processor_id())) 0.00 : ffff80001015edac: adrp x0, ffff80001151d000 0.00 : ffff80001015edb0: add x0, x0, #0x18 : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff80001015edb4: adrp x2, ffff80001190a000 <__per_cpu_offset+0x6d8> : invoke_rcu_core(): : { 0.00 : ffff80001015edb8: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001015edbc: mrs x1, tpidr_el1 : invoke_rcu_core(): : if (!cpu_online(smp_processor_id())) 0.00 : ffff80001015edc0: ldr w1, [x0, x1] : test_bit(): 0.00 : ffff80001015edc4: add x2, x2, #0x160 0.00 : ffff80001015edc8: add w0, w1, #0x3f 0.00 : ffff80001015edcc: cmp w1, #0x0 0.00 : ffff80001015edd0: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff80001015edd4: asr w0, w0, #6 0.00 : ffff80001015edd8: sxtw x0, w0 0.00 : ffff80001015eddc: ldr x0, [x2, x0, lsl #3] 0.00 : ffff80001015ede0: lsr x1, x0, x1 : invoke_rcu_core(): 0.00 : ffff80001015ede4: tbz w1, #0, ffff80001015ee64 : return; : if (use_softirq) 50.82 : ffff80001015ede8: adrp x0, ffff800011922000 49.18 : ffff80001015edec: ldrb w0, [x0, #2864] 0.00 : ffff80001015edf0: cbnz w0, ffff80001015ee6c 0.00 : ffff80001015edf4: str x19, [x29, #16] : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001015edf8: mrs x19, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff80001015edfc: and w0, w19, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff80001015ee00: cbz w0, ffff80001015ee8c : invoke_rcu_core_kthread(): : __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); 0.00 : ffff80001015ee04: adrp x1, ffff800011529000 0.00 : ffff80001015ee08: add x1, x1, #0xac0 : __my_cpu_offset(): : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff80001015ee0c: mov x2, sp : invoke_rcu_core_kthread(): 0.00 : ffff80001015ee10: add x0, x1, #0x104 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001015ee14: mrs x3, tpidr_el1 : invoke_rcu_core_kthread(): 0.00 : ffff80001015ee18: mov w4, #0x1 // #1 0.00 : ffff80001015ee1c: strb w4, [x0, x3] : t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); 0.00 : ffff80001015ee20: add x0, x1, #0xf8 : __my_cpu_offset(): 0.00 : ffff80001015ee24: mrs x2, tpidr_el1 : invoke_rcu_core_kthread(): 0.00 : ffff80001015ee28: ldr x0, [x0, x2] : if (t != NULL && t != current) 0.00 : ffff80001015ee2c: cbz x0, ffff80001015ee58 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001015ee30: mrs x3, sp_el0 : invoke_rcu_core_kthread(): 0.00 : ffff80001015ee34: cmp x0, x3 0.00 : ffff80001015ee38: b.eq ffff80001015ee58 // b.none : rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); 0.00 : ffff80001015ee3c: add x1, x1, #0x100 : rcu_wake_cond(): : if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current))) 0.00 : ffff80001015ee40: ldr w1, [x1, x2] 0.00 : ffff80001015ee44: cmp w1, #0x4 0.00 : ffff80001015ee48: b.ne ffff80001015ee54 // b.any : is_idle_task(): : * : * Return: 1 if @p is an idle task. 0 otherwise. : */ : static inline bool is_idle_task(const struct task_struct *p) : { : return !!(p->flags & PF_IDLE); 0.00 : ffff80001015ee4c: ldr w1, [x3, #44] : rcu_wake_cond(): 0.00 : ffff80001015ee50: tbz w1, #1, ffff80001015ee58 : wake_up_process(t); 0.00 : ffff80001015ee54: bl ffff800010116f00 : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff80001015ee58: msr daif, x19 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001015ee5c: nop 0.00 : ffff80001015ee60: ldr x19, [x29, #16] : invoke_rcu_core(): : raise_softirq(RCU_SOFTIRQ); : else : invoke_rcu_core_kthread(); : } 0.00 : ffff80001015ee64: ldp x29, x30, [sp], #32 0.00 : ffff80001015ee68: ret : raise_softirq(RCU_SOFTIRQ); 0.00 : ffff80001015ee6c: mov w0, #0x9 // #9 0.00 : ffff80001015ee70: bl ffff8000100ecde0 : } 0.00 : ffff80001015ee74: ldp x29, x30, [sp], #32 0.00 : ffff80001015ee78: ret : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff80001015ee7c: dsb sy 0.00 : ffff80001015ee80: ldr x19, [x29, #16] : invoke_rcu_core(): 0.00 : ffff80001015ee84: ldp x29, x30, [sp], #32 0.00 : ffff80001015ee88: ret : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff80001015ee8c: mov x0, #0x60 // #96 0.00 : ffff80001015ee90: msr daifset, #0x2 0.00 : ffff80001015ee94: b ffff80001015ee04 Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd24c8 : : arm64_preempt_schedule_irq(): : : core_initcall(tagged_addr_init); : #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */ : : asmlinkage void __sched arm64_preempt_schedule_irq(void) : { 50.28 : ffff800010cd24c8: stp x29, x30, [sp, #-16]! 0.00 : ffff800010cd24cc: mov x29, sp : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010cd24d0: b ffff800010cd24d8 : arm64_preempt_schedule_irq(): : * : * Only allow a task to be preempted once cpufeatures have been : * enabled. : */ : if (system_capabilities_finalized()) : preempt_schedule_irq(); 49.72 : ffff800010cd24d4: bl ffff800010cd2f48 : } 0.00 : ffff800010cd24d8: ldp x29, x30, [sp], #16 0.00 : ffff800010cd24dc: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001010e730 : : notifier_call_chain(): : * last notifier function called. : */ : static int notifier_call_chain(struct notifier_block **nl, : unsigned long val, void *v, : int nr_to_call, int *nr_calls) : { 49.13 : ffff80001010e730: stp x29, x30, [sp, #-64]! : int ret = NOTIFY_DONE; : struct notifier_block *nb, *next_nb; : : nb = rcu_dereference_raw(*nl); : : while (nb && nr_to_call) { 0.00 : ffff80001010e734: cmp w3, #0x0 : { 0.00 : ffff80001010e738: mov x29, sp 0.00 : ffff80001010e73c: str x23, [sp, #48] 0.00 : ffff80001010e740: mov x23, x1 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001010e744: ldr x1, [x0] : notifier_call_chain(): : while (nb && nr_to_call) { 0.00 : ffff80001010e748: ccmp x1, #0x0, #0x4, ne // ne = any 0.00 : ffff80001010e74c: b.eq ffff80001010e7bc // b.none 0.00 : ffff80001010e750: stp x19, x20, [x29, #16] 0.00 : ffff80001010e754: mov w19, w3 0.00 : ffff80001010e758: stp x21, x22, [x29, #32] 0.00 : ffff80001010e75c: mov x20, x4 0.00 : ffff80001010e760: mov x22, x2 0.00 : ffff80001010e764: nop : WARN(1, "Invalid notifier called!"); : nb = next_nb; : continue; : } : #endif : ret = nb->notifier_call(nb, val, v); 0.00 : ffff80001010e768: ldr x3, [x1] 0.00 : ffff80001010e76c: mov x0, x1 : __read_once_size(): 0.00 : ffff80001010e770: ldr x21, [x1, #8] : notifier_call_chain(): 0.00 : ffff80001010e774: mov x2, x22 0.00 : ffff80001010e778: mov x1, x23 : (*nr_calls)++; : : if (ret & NOTIFY_STOP_MASK) : break; : nb = next_nb; : nr_to_call--; 0.00 : ffff80001010e77c: sub w19, w19, #0x1 : ret = nb->notifier_call(nb, val, v); 0.00 : ffff80001010e780: blr x3 0.00 : ffff80001010e784: mov x1, x21 : if (nr_calls) 0.00 : ffff80001010e788: cbz x20, ffff80001010e798 : (*nr_calls)++; 0.00 : ffff80001010e78c: ldr w2, [x20] 0.00 : ffff80001010e790: add w2, w2, #0x1 0.00 : ffff80001010e794: str w2, [x20] : if (ret & NOTIFY_STOP_MASK) 0.00 : ffff80001010e798: tbnz w0, #15, ffff80001010e7a8 : while (nb && nr_to_call) { 0.00 : ffff80001010e79c: cmp x1, #0x0 0.00 : ffff80001010e7a0: ccmp w19, #0x0, #0x4, ne // ne = any 0.00 : ffff80001010e7a4: b.ne ffff80001010e768 // b.any 0.00 : ffff80001010e7a8: ldp x19, x20, [x29, #16] 0.00 : ffff80001010e7ac: ldp x21, x22, [x29, #32] : } : return ret; : } 50.87 : ffff80001010e7b0: ldr x23, [sp, #48] 0.00 : ffff80001010e7b4: ldp x29, x30, [sp], #64 0.00 : ffff80001010e7b8: ret : int ret = NOTIFY_DONE; 0.00 : ffff80001010e7bc: mov w0, #0x0 // #0 : return ret; 0.00 : ffff80001010e7c0: b ffff80001010e7b0 Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000102a1510 : : fget(): : { : return __fget(fd, FMODE_PATH, refs); : } : : struct file *fget(unsigned int fd) : { 0.00 : ffff8000102a1510: stp x29, x30, [sp, #-16]! : __fget(): : return __fget_files(current->files, fd, mask, refs); 0.00 : ffff8000102a1514: mov w1, w0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 100.00 : ffff8000102a1518: mrs x2, sp_el0 : fget(): : { 0.00 : ffff8000102a151c: mov x29, sp : __fget(): : return __fget_files(current->files, fd, mask, refs); 0.00 : ffff8000102a1520: ldr x0, [x2, #1624] 0.00 : ffff8000102a1524: mov w3, #0x1 // #1 0.00 : ffff8000102a1528: mov w2, #0x4000 // #16384 0.00 : ffff8000102a152c: bl ffff8000102a1428 <__fget_files> : fget(): : return __fget(fd, FMODE_PATH, 1); : } 0.00 : ffff8000102a1530: ldp x29, x30, [sp], #16 0.00 : ffff8000102a1534: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010b38e38 : : armpmu_disable(): : if (enabled) : armpmu->start(armpmu); : } : : static void armpmu_disable(struct pmu *pmu) : { 0.00 : ffff800010b38e38: stp x29, x30, [sp, #-16]! : struct arm_pmu *armpmu = to_arm_pmu(pmu); : : /* For task-bound events we may be called on other CPUs */ : if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) 0.00 : ffff800010b38e3c: adrp x1, ffff80001151d000 0.00 : ffff800010b38e40: add x1, x1, #0x18 : cpumask_test_cpu(): : * : * Returns 1 if @cpu is set in @cpumask, else returns 0 : */ : static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask) : { : return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); 0.00 : ffff800010b38e44: add x4, x0, #0x128 : armpmu_disable(): : { 0.00 : ffff800010b38e48: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010b38e4c: mrs x2, tpidr_el1 : armpmu_disable(): : if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) 0.00 : ffff800010b38e50: ldr w2, [x1, x2] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010b38e54: add w1, w2, #0x3f 0.00 : ffff800010b38e58: cmp w2, #0x0 0.00 : ffff800010b38e5c: csel w1, w1, w2, lt // lt = tstop 50.30 : ffff800010b38e60: asr w1, w1, #6 0.00 : ffff800010b38e64: sxtw x1, w1 0.00 : ffff800010b38e68: ldr x1, [x4, x1, lsl #3] 0.00 : ffff800010b38e6c: lsr x2, x1, x2 : armpmu_disable(): 0.00 : ffff800010b38e70: tbz w2, #0, ffff800010b38e7c : return; : : armpmu->stop(armpmu); 49.70 : ffff800010b38e74: ldr x1, [x0, #408] 0.00 : ffff800010b38e78: blr x1 : } 0.00 : ffff800010b38e7c: ldp x29, x30, [sp], #16 0.00 : ffff800010b38e80: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101b44c8 : : irq_work_run_list(): : : return true; : } : : static void irq_work_run_list(struct llist_head *list) : { 0.00 : ffff8000101b44c8: stp x29, x30, [sp, #-48]! 0.00 : ffff8000101b44cc: mov x29, sp : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000101b44d0: mrs x2, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000101b44d4: and w1, w2, #0x80 : irq_work_run_list(): : struct irq_work *work, *tmp; : struct llist_node *llnode; : : BUG_ON(!irqs_disabled()); 0.00 : ffff8000101b44d8: cbz w1, ffff8000101b45a0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101b44dc: ldr x1, [x0] : irq_work_run_list(): : : if (llist_empty(list)) 0.00 : ffff8000101b44e0: cbnz x1, ffff8000101b44ec : * no-one else claimed it meanwhile. : */ : flags &= ~IRQ_WORK_PENDING; : (void)atomic_cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY); : } : } 100.00 : ffff8000101b44e4: ldp x29, x30, [sp], #48 0.00 : ffff8000101b44e8: ret : __xchg_case_mb_64(): : __XCHG_CASE(w, , rel_, 32, , , , , l, "memory") : __XCHG_CASE( , , rel_, 64, , , , , l, "memory") : __XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory") : __XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory") : __XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory") : __XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory") 0.00 : ffff8000101b44ec: mov x1, #0x0 // #0 0.00 : ffff8000101b44f0: str x19, [x29, #16] 0.00 : ffff8000101b44f4: str x21, [x29, #32] 0.00 : ffff8000101b44f8: prfm pstl1strm, [x0] 0.00 : ffff8000101b44fc: ldxr x2, [x0] 0.00 : ffff8000101b4500: stlxr w3, x1, [x0] 0.00 : ffff8000101b4504: cbnz w3, ffff8000101b44fc 0.00 : ffff8000101b4508: dmb ish : __ll_sc_atomic_fetch_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC_OPS(andnot, bic, ) 0.00 : ffff8000101b450c: mov w21, #0x1 // #1 : irq_work_run_list(): : llist_for_each_entry_safe(work, tmp, llnode, llnode) { 0.00 : ffff8000101b4510: sub x19, x2, #0x8 0.00 : ffff8000101b4514: cbz x2, ffff8000101b45ac 0.00 : ffff8000101b4518: str x20, [x29, #24] 0.00 : ffff8000101b451c: str x22, [x29, #40] 0.00 : ffff8000101b4520: ldr x22, [x19, #8] 0.00 : ffff8000101b4524: sub x22, x22, #0x8 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101b4528: b ffff8000101b4594 0.00 : ffff8000101b452c: b ffff8000101b4594 : __lse_atomic_fetch_andnot(): : ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \ : ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC_FETCH_OPS(andnot, ldclr) 0.00 : ffff8000101b4530: mov w2, w21 0.00 : ffff8000101b4534: ldclral w2, w2, [x19] 0.00 : ffff8000101b4538: mov w20, w2 : irq_work_run_list(): : work->func(work); 0.00 : ffff8000101b453c: ldr x1, [x19, #16] 0.00 : ffff8000101b4540: mov x0, x19 0.00 : ffff8000101b4544: blr x1 : flags &= ~IRQ_WORK_PENDING; 0.00 : ffff8000101b4548: and w1, w20, #0xfffffffe : (void)atomic_cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY); 0.00 : ffff8000101b454c: and w2, w20, #0xfffffffc : atomic_cmpxchg(): : #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) : static inline int : atomic_cmpxchg(atomic_t *v, int old, int new) : { : kasan_check_write(v, sizeof(*v)); : return arch_atomic_cmpxchg(v, old, new); 0.00 : ffff8000101b4550: sxtw x0, w1 : arch_static_branch_jump(): 0.00 : ffff8000101b4554: b ffff8000101b4588 0.00 : ffff8000101b4558: b ffff8000101b4588 : __lse__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 0.00 : ffff8000101b455c: mov x0, x19 0.00 : ffff8000101b4560: mov w3, w1 0.00 : ffff8000101b4564: casal w3, w2, [x19] 0.00 : ffff8000101b4568: mov w0, w3 0.00 : ffff8000101b456c: mov x19, x22 : irq_work_run_list(): : llist_for_each_entry_safe(work, tmp, llnode, llnode) { 0.00 : ffff8000101b4570: cmn x22, #0x8 0.00 : ffff8000101b4574: b.ne ffff8000101b4520 // b.any 0.00 : ffff8000101b4578: ldp x19, x20, [x29, #16] 0.00 : ffff8000101b457c: ldp x21, x22, [x29, #32] : } 0.00 : ffff8000101b4580: ldp x29, x30, [sp], #48 0.00 : ffff8000101b4584: ret : __ll_sc__cmpxchg_case_mb_32(): : __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 0.00 : ffff8000101b4588: and x0, x0, #0xffffffff 0.00 : ffff8000101b458c: b ffff8000101b48f8 0.00 : ffff8000101b4590: b ffff8000101b456c : __ll_sc_atomic_fetch_andnot(): : ATOMIC_OPS(andnot, bic, ) 0.00 : ffff8000101b4594: b ffff8000101b4918 0.00 : ffff8000101b4598: mov w20, w2 0.00 : ffff8000101b459c: b ffff8000101b453c 0.00 : ffff8000101b45a0: stp x19, x20, [x29, #16] 0.00 : ffff8000101b45a4: stp x21, x22, [x29, #32] : irq_work_run_list(): : BUG_ON(!irqs_disabled()); 0.00 : ffff8000101b45a8: brk #0x800 0.00 : ffff8000101b45ac: ldr x19, [x29, #16] 0.00 : ffff8000101b45b0: ldr x21, [x29, #32] 0.00 : ffff8000101b45b4: b ffff8000101b44e4 Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015ccd0 : : rcu_gp_kthread_wake(): : * pre-sleep check of the awaken condition. In this case, a wakeup really : * is required, and is therefore supplied. : */ : static void rcu_gp_kthread_wake(void) : { : if ((current == rcu_state.gp_kthread && 0.00 : ffff80001015ccd0: adrp x0, ffff800011922000 0.00 : ffff80001015ccd4: add x2, x0, #0xc80 : { 0.00 : ffff80001015ccd8: stp x29, x30, [sp, #-16]! : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001015ccdc: mrs x1, sp_el0 : rcu_gp_kthread_wake(): 0.00 : ffff80001015cce0: mov x29, sp : if ((current == rcu_state.gp_kthread && 0.00 : ffff80001015cce4: ldr x2, [x2, #8784] 0.00 : ffff80001015cce8: cmp x2, x1 0.00 : ffff80001015ccec: b.eq ffff80001015cd14 // b.none : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001015ccf0: add x1, x0, #0xc80 : rcu_gp_kthread_wake(): : !in_irq() && !in_serving_softirq()) || : !READ_ONCE(rcu_state.gp_flags) || 0.00 : ffff80001015ccf4: cmp x2, #0x0 : __read_once_size(): 0.00 : ffff80001015ccf8: add x0, x1, #0x2, lsl #12 0.00 : ffff80001015ccfc: ldrh w0, [x0, #624] 0.00 : ffff80001015cd00: and w0, w0, #0xffff : rcu_gp_kthread_wake(): 0.00 : ffff80001015cd04: ccmp w0, #0x0, #0x4, ne // ne = any 0.00 : ffff80001015cd08: b.ne ffff80001015cd2c // b.any : !rcu_state.gp_kthread) : return; : WRITE_ONCE(rcu_state.gp_wake_time, jiffies); : WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq)); : swake_up_one(&rcu_state.gp_wq); : } 0.00 : ffff80001015cd0c: ldp x29, x30, [sp], #16 0.00 : ffff80001015cd10: ret : __read_once_size(): 0.00 : ffff80001015cd14: ldr w1, [x2, #16] : rcu_gp_kthread_wake(): : if ((current == rcu_state.gp_kthread && 0.00 : ffff80001015cd18: tst w1, #0xf0000 0.00 : ffff80001015cd1c: b.ne ffff80001015ccf0 // b.any : __read_once_size(): 0.00 : ffff80001015cd20: ldr w1, [x2, #16] : rcu_gp_kthread_wake(): : !in_irq() && !in_serving_softirq()) || 0.00 : ffff80001015cd24: tbnz w1, #8, ffff80001015ccf0 0.00 : ffff80001015cd28: b ffff80001015cd0c : WRITE_ONCE(rcu_state.gp_wake_time, jiffies); 49.51 : ffff80001015cd2c: adrp x2, ffff800011907000 : swake_up_one(&rcu_state.gp_wq); 0.00 : ffff80001015cd30: adrp x0, ffff800011924000 0.00 : ffff80001015cd34: add x0, x0, #0xed8 : WRITE_ONCE(rcu_state.gp_wake_time, jiffies); 50.49 : ffff80001015cd38: ldr x2, [x2, #2432] : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001015cd3c: str x2, [x1, #8824] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001015cd40: ldr x2, [x1, #8776] : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001015cd44: str x2, [x1, #8832] : rcu_gp_kthread_wake(): : swake_up_one(&rcu_state.gp_wq); 0.00 : ffff80001015cd48: bl ffff8000101306e8 : } 0.00 : ffff80001015cd4c: ldp x29, x30, [sp], #16 0.00 : ffff80001015cd50: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001017b670 : : tick_sched_handle(): : : if (ts->inidle) : ts->got_idle_tick = 1; : } : : static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) 0.00 : ffff80001017b670: stp x29, x30, [sp, #-16]! 0.00 : ffff80001017b674: mov x29, sp : * time. This happens on complete idle SMP systems while : * waiting on the login prompt. We also increment the "start of : * idle" jiffy stamp so the idle accounting adjustment we do : * when we go busy again does not account too much ticks. : */ : if (ts->tick_stopped) { 47.21 : ffff80001017b678: ldrb w2, [x0, #76] 0.00 : ffff80001017b67c: tbz w2, #1, ffff80001017b690 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001017b680: mrs x2, sp_el0 : is_idle_task(): : * : * Return: 1 if @p is an idle task. 0 otherwise. : */ : static inline bool is_idle_task(const struct task_struct *p) : { : return !!(p->flags & PF_IDLE); 0.00 : ffff80001017b684: ldr w2, [x2, #44] : tick_sched_handle(): : touch_softlockup_watchdog_sched(); : if (is_idle_task(current)) 0.00 : ffff80001017b688: tbnz w2, #1, ffff80001017b6b0 : /* : * In case the current tick fired too early past its expected : * expiration, make sure we don't bypass the next clock reprogramming : * to the same deadline. : */ : ts->next_tick = 0; 0.00 : ffff80001017b68c: str xzr, [x0, #88] : } : #endif : update_process_times(user_mode(regs)); 52.79 : ffff80001017b690: ldr x0, [x1] 0.00 : ffff80001017b694: tst x0, #0xf 0.00 : ffff80001017b698: cset w0, eq // eq = none 0.00 : ffff80001017b69c: bl ffff80001016aa20 : profile_tick(CPU_PROFILING); 0.00 : ffff80001017b6a0: mov w0, #0x1 // #1 0.00 : ffff80001017b6a4: bl ffff800010166f30 : } 0.00 : ffff80001017b6a8: ldp x29, x30, [sp], #16 0.00 : ffff80001017b6ac: ret : ts->idle_jiffies++; 0.00 : ffff80001017b6b0: ldr x2, [x0, #96] 0.00 : ffff80001017b6b4: add x2, x2, #0x1 0.00 : ffff80001017b6b8: str x2, [x0, #96] 0.00 : ffff80001017b6bc: b ffff80001017b68c Percent | Source code & Disassembly of vmlinux for cycles (4 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001012a490 : : update_rt_migration(): : cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask); : } : : static void update_rt_migration(struct rt_rq *rt_rq) : { : if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) { 0.00 : ffff80001012a490: ldr x2, [x0, #1632] 0.00 : ffff80001012a494: ldr w1, [x0, #1648] 0.00 : ffff80001012a498: cbz x2, ffff80001012a4c0 0.00 : ffff80001012a49c: ldr x2, [x0, #1640] 0.00 : ffff80001012a4a0: cmp x2, #0x1 0.00 : ffff80001012a4a4: b.ls ffff80001012a4c0 // b.plast : if (!rt_rq->overloaded) { 0.00 : ffff80001012a4a8: cbnz w1, ffff80001012a4bc : rt_set_overload(): : if (!rq->online) 0.00 : ffff80001012a4ac: ldr w1, [x0, #2060] 0.00 : ffff80001012a4b0: cbnz w1, ffff80001012a540 : update_rt_migration(): : rt_set_overload(rq_of_rt_rq(rt_rq)); : rt_rq->overloaded = 1; 0.00 : ffff80001012a4b4: mov w1, #0x1 // #1 0.00 : ffff80001012a4b8: str w1, [x0, #1648] : } : } else if (rt_rq->overloaded) { : rt_clear_overload(rq_of_rt_rq(rt_rq)); : rt_rq->overloaded = 0; : } : } 100.00 : ffff80001012a4bc: ret : } else if (rt_rq->overloaded) { 0.00 : ffff80001012a4c0: cbz w1, ffff80001012a4bc : rt_clear_overload(): : if (!rq->online) 0.00 : ffff80001012a4c4: sub x1, x0, #0x200 0.00 : ffff80001012a4c8: ldr w2, [x1, #2572] 0.00 : ffff80001012a4cc: cbnz w2, ffff80001012a4d8 : update_rt_migration(): : rt_rq->overloaded = 0; 0.00 : ffff80001012a4d0: str wzr, [x0, #1648] : } 0.00 : ffff80001012a4d4: ret : rt_clear_overload(): : atomic_dec(&rq->rd->rto_count); 0.00 : ffff80001012a4d8: ldr x2, [x1, #2464] 0.00 : ffff80001012a4dc: add x4, x2, #0x4 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001012a4e0: b ffff80001012a524 0.00 : ffff80001012a4e4: b ffff80001012a524 : __lse_atomic_sub(): : : #undef ATOMIC_FETCH_OP_AND : : static inline void __lse_atomic_sub(int i, atomic_t *v) : { : asm volatile( 0.00 : ffff80001012a4e8: mov w3, #0x1 // #1 0.00 : ffff80001012a4ec: neg w3, w3 0.00 : ffff80001012a4f0: stadd w3, [x4] : cpumask_clear_cpu(): : * @cpu: cpu number (< nr_cpu_ids) : * @dstp: the cpumask pointer : */ : static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) : { : clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 0.00 : ffff80001012a4f4: ldr w4, [x1, #2568] : clear_bit(): : } : : static inline void clear_bit(unsigned int nr, volatile unsigned long *p) : { : p += BIT_WORD(nr); : atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff80001012a4f8: mov x2, #0x1 // #1 : cpumask_clear_cpu(): 0.00 : ffff80001012a4fc: ldr x1, [x1, #2464] 0.00 : ffff80001012a500: add x3, x1, #0x100 : clear_bit(): : p += BIT_WORD(nr); 0.00 : ffff80001012a504: lsr w1, w4, #6 : atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff80001012a508: lsl x2, x2, x4 : p += BIT_WORD(nr); 0.00 : ffff80001012a50c: add x1, x3, x1, lsl #3 : arch_static_branch_jump(): 0.00 : ffff80001012a510: b ffff80001012a534 0.00 : ffff80001012a514: b ffff80001012a534 : __lse_atomic64_andnot(): : " " #asm_op " %[i], %[v]\n" \ : : [i] "+r" (i), [v] "+Q" (v->counter) \ : : "r" (v)); \ : } : : ATOMIC64_OP(andnot, stclr) 0.00 : ffff80001012a518: stclr x2, [x1] : update_rt_migration(): : rt_rq->overloaded = 0; 0.00 : ffff80001012a51c: str wzr, [x0, #1648] 0.00 : ffff80001012a520: b ffff80001012a4d4 : __ll_sc_atomic_sub(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff80001012a524: mov w3, #0x1 // #1 0.00 : ffff80001012a528: add x2, x2, #0x4 0.00 : ffff80001012a52c: b ffff80001012b46c 0.00 : ffff80001012a530: b ffff80001012a4f4 : __ll_sc_atomic64_andnot(): : /* : * GAS converts the mysterious and undocumented BIC (immediate) alias to : * an AND (immediate) instruction with the immediate inverted. We don't : * have a constraint for this, so fall back to register. : */ : ATOMIC64_OPS(andnot, bic, ) 0.00 : ffff80001012a534: b ffff80001012b484 : update_rt_migration(): 0.00 : ffff80001012a538: str wzr, [x0, #1648] 0.00 : ffff80001012a53c: b ffff80001012a4d4 : rt_set_overload(): : cpumask_set_cpu(rq->cpu, rq->rd->rto_mask); 0.00 : ffff80001012a540: ldr w4, [x0, #2056] : set_bit(): : atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff80001012a544: mov x2, #0x1 // #1 : cpumask_set_cpu(): : set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 0.00 : ffff80001012a548: ldr x1, [x0, #1952] 0.00 : ffff80001012a54c: add x3, x1, #0x100 : set_bit(): : p += BIT_WORD(nr); 0.00 : ffff80001012a550: lsr w1, w4, #6 : atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p); 0.00 : ffff80001012a554: lsl x2, x2, x4 : p += BIT_WORD(nr); 0.00 : ffff80001012a558: add x1, x3, x1, lsl #3 : arch_static_branch_jump(): 0.00 : ffff80001012a55c: b ffff80001012a56c 0.00 : ffff80001012a560: b ffff80001012a56c : __lse_atomic64_or(): : ATOMIC64_OP(or, stset) 0.00 : ffff80001012a564: stset x2, [x1] 0.00 : ffff80001012a568: b ffff80001012a570 : __ll_sc_atomic64_or(): : ATOMIC64_OPS(or, orr, L) 0.00 : ffff80001012a56c: b ffff80001012b49c : rt_set_overload(): : smp_wmb(); 0.00 : ffff80001012a570: dmb ishst : atomic_inc(&rq->rd->rto_count); 0.00 : ffff80001012a574: ldr x1, [x0, #1952] 0.00 : ffff80001012a578: add x3, x1, #0x4 : arch_static_branch_jump(): 0.00 : ffff80001012a57c: b ffff80001012a590 0.00 : ffff80001012a580: b ffff80001012a590 : __lse_atomic_add(): : ATOMIC_OP(add, stadd) 0.00 : ffff80001012a584: mov w2, #0x1 // #1 0.00 : ffff80001012a588: stadd w2, [x3] 0.00 : ffff80001012a58c: b ffff80001012a4b4 : __ll_sc_atomic_add(): : ATOMIC_OPS(add, add, I) 0.00 : ffff80001012a590: add x1, x1, #0x4 0.00 : ffff80001012a594: b ffff80001012b4b4 0.00 : ffff80001012a598: b ffff80001012a4b4 Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010105988 : : queue_delayed_work_on(): : * @delay is zero and @dwork is idle, it will be scheduled for immediate : * execution. : */ : bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq, : struct delayed_work *dwork, unsigned long delay) : { 0.00 : ffff800010105988: stp x29, x30, [sp, #-48]! 0.00 : ffff80001010598c: mov x29, sp 0.00 : ffff800010105990: stp x19, x20, [sp, #16] 0.00 : ffff800010105994: adrp x19, ffff800011909000 0.00 : ffff800010105998: add x4, x19, #0x908 0.00 : ffff80001010599c: ldr x5, [x4] 0.00 : ffff8000101059a0: str x5, [x29, #40] 0.00 : ffff8000101059a4: mov x5, #0x0 // #0 : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000101059a8: mrs x20, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff8000101059ac: and w4, w20, #0x80 : arch_local_irq_save(): : : /* : * There are too many states with IRQs disabled, just keep the current : * state if interrupts are already disabled/masked. : */ : if (!arch_irqs_disabled_flags(flags)) 0.00 : ffff8000101059b0: cbnz w4, ffff8000101059bc : arch_local_irq_disable(): : asm volatile(ALTERNATIVE( 0.00 : ffff8000101059b4: mov x4, #0x60 // #96 0.00 : ffff8000101059b8: msr daifset, #0x2 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101059bc: ldr x4, [x2] : test_and_set_bit(): : { : long old; : unsigned long mask = BIT_MASK(nr); : : p += BIT_WORD(nr); : if (READ_ONCE(*p) & mask) 0.00 : ffff8000101059c0: tbz w4, #0, ffff8000101059f8 : queue_delayed_work_on(): : struct work_struct *work = &dwork->work; : bool ret = false; 0.00 : ffff8000101059c4: mov w0, #0x0 // #0 : arch_local_irq_restore(): : /* : * restore saved IRQ state : */ : static inline void arch_local_irq_restore(unsigned long flags) : { : asm volatile(ALTERNATIVE( 0.00 : ffff8000101059c8: msr daif, x20 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101059cc: nop : queue_delayed_work_on(): : ret = true; : } : : local_irq_restore(flags); : return ret; : } 0.00 : ffff8000101059d0: add x19, x19, #0x908 0.00 : ffff8000101059d4: ldr x2, [x29, #40] 0.00 : ffff8000101059d8: ldr x1, [x19] 0.00 : ffff8000101059dc: eor x1, x2, x1 0.00 : ffff8000101059e0: cbnz x1, ffff800010105a20 0.00 : ffff8000101059e4: ldp x19, x20, [sp, #16] 0.00 : ffff8000101059e8: ldp x29, x30, [sp], #48 0.00 : ffff8000101059ec: ret : arch_local_irq_restore(): : ARM64_HAS_IRQ_PRIO_MASKING) : : : : "r" (flags) : : "memory"); : : pmr_sync(); 0.00 : ffff8000101059f0: dsb sy 100.00 : ffff8000101059f4: b ffff8000101059d0 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101059f8: b ffff800010105a18 0.00 : ffff8000101059fc: b ffff800010105a18 : __lse_atomic64_fetch_or(): : ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ : ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") : : ATOMIC64_FETCH_OPS(andnot, ldclr) : ATOMIC64_FETCH_OPS(or, ldset) 0.00 : ffff800010105a00: mov x4, #0x1 // #1 0.00 : ffff800010105a04: ldsetal x4, x4, [x2] : queue_delayed_work_on(): : if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { 0.00 : ffff800010105a08: tbnz w4, #0, ffff8000101059c4 : __queue_delayed_work(cpu, wq, dwork, delay); 0.00 : ffff800010105a0c: bl ffff8000101058c0 <__queue_delayed_work> : ret = true; 0.00 : ffff800010105a10: mov w0, #0x1 // #1 0.00 : ffff800010105a14: b ffff8000101059c8 : __ll_sc_atomic64_fetch_or(): : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(and, and, L) : ATOMIC64_OPS(or, orr, L) 0.00 : ffff800010105a18: b ffff800010108fc8 0.00 : ffff800010105a1c: b ffff800010105a08 : queue_delayed_work_on(): : } 0.00 : ffff800010105a20: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000104569f8 : : bvec_free(): : return bvec_slabs[--idx].nr_vecs; : } : : void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx) : { : if (!idx) 49.94 : ffff8000104569f8: cbz w2, ffff800010456a34 : { 0.00 : ffff8000104569fc: stp x29, x30, [sp, #-16]! : return; : idx--; 0.00 : ffff800010456a00: sub w2, w2, #0x1 : : BIO_BUG_ON(idx >= BVEC_POOL_NR); 0.00 : ffff800010456a04: cmp w2, #0x5 : { 0.00 : ffff800010456a08: mov x29, sp : BIO_BUG_ON(idx >= BVEC_POOL_NR); 0.00 : ffff800010456a0c: b.hi ffff800010456a38 // b.pmore : : if (idx == BVEC_POOL_MAX) { 0.00 : ffff800010456a10: b.eq ffff800010456a3c // b.none : mempool_free(bv, pool); : } else { : struct biovec_slab *bvs = bvec_slabs + idx; 0.00 : ffff800010456a14: adrp x0, ffff80001190c000 0.00 : ffff800010456a18: add x0, x0, #0x8c8 0.00 : ffff800010456a1c: mov w3, #0x18 // #24 : : kmem_cache_free(bvs->slab, bv); 0.00 : ffff800010456a20: umaddl x2, w2, w3, x0 0.00 : ffff800010456a24: ldr x0, [x2, #16] 0.00 : ffff800010456a28: bl ffff800010253518 : } : } 0.00 : ffff800010456a2c: ldp x29, x30, [sp], #16 0.00 : ffff800010456a30: ret 50.06 : ffff800010456a34: ret : BIO_BUG_ON(idx >= BVEC_POOL_NR); 0.00 : ffff800010456a38: brk #0x800 0.00 : ffff800010456a3c: mov x3, x1 : mempool_free(bv, pool); 0.00 : ffff800010456a40: mov x1, x0 0.00 : ffff800010456a44: mov x0, x3 0.00 : ffff800010456a48: bl ffff8000101d7548 : } 0.00 : ffff800010456a4c: ldp x29, x30, [sp], #16 0.00 : ffff800010456a50: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010162478 : : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010162478: mrs x2, tpidr_el1 : rcu_needs_cpu(): : * CPU has RCU callbacks queued. : */ : int rcu_needs_cpu(u64 basemono, u64 *nextevt) : { : *nextevt = KTIME_MAX; : return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && 100.00 : ffff80001016247c: adrp x0, ffff800011529000 0.00 : ffff800010162480: add x0, x0, #0xac0 0.00 : ffff800010162484: add x2, x0, x2 : *nextevt = KTIME_MAX; 0.00 : ffff800010162488: mov x3, #0x7fffffffffffffff // #9223372036854775807 0.00 : ffff80001016248c: str x3, [x1] : return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && 0.00 : ffff800010162490: mov w0, #0x0 // #0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010162494: ldr x1, [x2, #80] : rcu_needs_cpu(): 0.00 : ffff800010162498: cbz x1, ffff8000101624a8 0.00 : ffff80001016249c: ldrb w0, [x2, #161] 0.00 : ffff8000101624a0: cmp w0, #0x0 0.00 : ffff8000101624a4: cset w0, eq // eq = none : !rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist); : } 0.00 : ffff8000101624a8: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010122e68 : : can_migrate_task(): : /* : * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? : */ : static : int can_migrate_task(struct task_struct *p, struct lb_env *env) : { 0.00 : ffff800010122e68: stp x29, x30, [sp, #-80]! 0.00 : ffff800010122e6c: mov x29, sp 0.00 : ffff800010122e70: stp x19, x20, [sp, #16] 0.00 : ffff800010122e74: mov x19, x1 0.00 : ffff800010122e78: str x21, [sp, #32] 0.00 : ffff800010122e7c: mov x20, x0 : * 1) throttled_lb_pair, or : * 2) cannot be migrated to this CPU due to cpus_ptr, or : * 3) running (obviously), or : * 4) are cache-hot on their current CPU. : */ : if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) 0.00 : ffff800010122e80: ldr w1, [x1, #20] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010122e84: ldr x3, [x0, #736] 0.00 : ffff800010122e88: cmp w1, #0x0 0.00 : ffff800010122e8c: add w0, w1, #0x3f 0.00 : ffff800010122e90: csel w0, w0, w1, lt // lt = tstop 0.00 : ffff800010122e94: ldr w2, [x19, #64] 0.00 : ffff800010122e98: asr w0, w0, #6 0.00 : ffff800010122e9c: sxtw x0, w0 0.00 : ffff800010122ea0: ldr x0, [x3, x0, lsl #3] 0.00 : ffff800010122ea4: lsr x0, x0, x1 : can_migrate_task(): : return 0; : : if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 0.00 : ffff800010122ea8: and w21, w0, #0x1 0.00 : ffff800010122eac: tbnz w0, #0, ffff800010122f38 : int cpu; : : schedstat_inc(p->se.statistics.nr_failed_migrations_affine); : : env->flags |= LBF_SOME_PINNED; 0.00 : ffff800010122eb0: orr w1, w2, #0x8 : * meet load balance goals by pulling other tasks on src_cpu. : * : * Avoid computing new_dst_cpu for NEWLY_IDLE or if we have : * already computed one in current iteration. : */ : if (env->idle == CPU_NEWLY_IDLE || (env->flags & LBF_DST_PINNED)) 0.00 : ffff800010122eb4: ldr w0, [x19, #44] : env->flags |= LBF_SOME_PINNED; 0.00 : ffff800010122eb8: str w1, [x19, #64] : if (env->idle == CPU_NEWLY_IDLE || (env->flags & LBF_DST_PINNED)) 0.00 : ffff800010122ebc: cmp w0, #0x2 0.00 : ffff800010122ec0: b.eq ffff800010122f20 // b.none 0.00 : ffff800010122ec4: tbnz w2, #2, ffff800010122f20 0.00 : ffff800010122ec8: str x22, [x29, #40] : return 0; : : /* Prevent to re-select dst_cpu via env's CPUs: */ : for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { 0.00 : ffff800010122ecc: mov w0, #0xffffffff // #-1 0.00 : ffff800010122ed0: adrp x22, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010122ed4: add x22, x22, #0x2f0 0.00 : ffff800010122ed8: b ffff800010122efc : test_bit(): 0.00 : ffff800010122edc: cmp w0, #0x0 0.00 : ffff800010122ee0: ldr x2, [x20, #736] 0.00 : ffff800010122ee4: csel w1, w1, w0, lt // lt = tstop 0.00 : ffff800010122ee8: asr w1, w1, #6 0.00 : ffff800010122eec: sxtw x1, w1 0.00 : ffff800010122ef0: ldr x1, [x2, x1, lsl #3] 0.00 : ffff800010122ef4: lsr x1, x1, x0 : can_migrate_task(): : if (cpumask_test_cpu(cpu, p->cpus_ptr)) { 0.00 : ffff800010122ef8: tbnz w1, #0, ffff8000101230f0 : for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { 54.17 : ffff800010122efc: ldr x1, [x19, #32] 0.00 : ffff800010122f00: ldr x2, [x19, #56] 0.00 : ffff800010122f04: bl ffff800010cb93f0 : test_bit(): 0.00 : ffff800010122f08: add w1, w0, #0x3f : can_migrate_task(): 0.00 : ffff800010122f0c: ldr w2, [x22] 0.00 : ffff800010122f10: cmp w0, w2 0.00 : ffff800010122f14: b.cc ffff800010122edc // b.lo, b.ul, b.last 45.83 : ffff800010122f18: ldr x22, [x29, #40] 0.00 : ffff800010122f1c: nop : return 0; 0.00 : ffff800010122f20: mov w21, #0x0 // #0 : return 1; : } : : schedstat_inc(p->se.statistics.nr_failed_migrations_hot); : return 0; : } 0.00 : ffff800010122f24: mov w0, w21 0.00 : ffff800010122f28: ldr x21, [sp, #32] 0.00 : ffff800010122f2c: ldp x19, x20, [sp, #16] 0.00 : ffff800010122f30: ldp x29, x30, [sp], #80 0.00 : ffff800010122f34: ret : env->flags &= ~LBF_ALL_PINNED; 0.00 : ffff800010122f38: and w2, w2, #0xfffffffe 0.00 : ffff800010122f3c: str w2, [x19, #64] : if (task_running(env->src_rq, p)) { 0.00 : ffff800010122f40: ldr w0, [x20, #64] 0.00 : ffff800010122f44: cbnz w0, ffff800010122f20 0.00 : ffff800010122f48: str x24, [x29, #56] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010122f4c: ldr x24, [x20, #2248] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff800010122f50: b ffff8000101230d0 : migrate_degrades_locality(): : if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 0.00 : ffff800010122f54: ldr x0, [x20, #2256] 0.00 : ffff800010122f58: cbz x0, ffff8000101230d0 0.00 : ffff800010122f5c: ldr x3, [x19] 0.00 : ffff800010122f60: ldr w0, [x3, #56] 0.00 : ffff800010122f64: tbz w0, #14, ffff8000101230d0 0.00 : ffff800010122f68: stp x22, x23, [x29, #40] : cpu_to_node(): : #endif : : #ifndef cpu_to_node : static inline int cpu_to_node(int cpu) : { : return per_cpu(numa_node, cpu); 0.00 : ffff800010122f6c: adrp x2, ffff800011909000 0.00 : ffff800010122f70: add x2, x2, #0x928 0.00 : ffff800010122f74: adrp x0, ffff800011526000 0.00 : ffff800010122f78: ldrsw x4, [x19, #16] 0.00 : ffff800010122f7c: add x0, x0, #0xd58 0.00 : ffff800010122f80: ldr x1, [x2, w1, sxtw #3] 0.00 : ffff800010122f84: ldr x4, [x2, x4, lsl #3] 0.00 : ffff800010122f88: ldr w23, [x1, x0] 0.00 : ffff800010122f8c: ldr w22, [x4, x0] : migrate_degrades_locality(): : if (src_nid == dst_nid) 0.00 : ffff800010122f90: cmp w22, w23 0.00 : ffff800010122f94: b.eq ffff8000101230c8 // b.none : if (src_nid == p->numa_preferred_nid) { 0.00 : ffff800010122f98: ldr w0, [x20, #2192] 0.00 : ffff800010122f9c: cmp w22, w0 0.00 : ffff800010122fa0: b.eq ffff800010123108 // b.none : if (dst_nid == p->numa_preferred_nid) 0.00 : ffff800010122fa4: cmp w23, w0 0.00 : ffff800010122fa8: b.eq ffff8000101231d8 // b.none : if (env->idle == CPU_IDLE) 0.00 : ffff800010122fac: ldr w0, [x19, #44] 0.00 : ffff800010122fb0: cbz w0, ffff8000101230c8 0.00 : ffff800010122fb4: str x25, [x29, #64] : dist = node_distance(src_nid, dst_nid); 0.00 : ffff800010122fb8: mov w1, w23 0.00 : ffff800010122fbc: mov w0, w22 0.00 : ffff800010122fc0: bl ffff8000100a6118 <__node_distance> 0.00 : ffff800010122fc4: mov w25, w0 : if (numa_group) { 0.00 : ffff800010122fc8: cbz x24, ffff800010123138 : __read_once_size(): 0.00 : ffff800010122fcc: ldr x0, [x20, #2248] : group_weight(): : return 0; 0.00 : ffff800010122fd0: mov x24, #0x0 // #0 : if (!ng) 0.00 : ffff800010122fd4: cbz x0, ffff800010123048 : total_faults = ng->total_faults; 0.00 : ffff800010122fd8: ldr x24, [x0, #40] : if (!total_faults) 0.00 : ffff800010122fdc: cbz x24, ffff800010123048 0.00 : ffff800010122fe0: str x26, [x29, #72] : group_faults(): : return 0; 0.00 : ffff800010122fe4: mov x26, #0x0 // #0 : __read_once_size(): 0.00 : ffff800010122fe8: ldr x0, [x20, #2248] : group_faults(): : if (!ng) 0.00 : ffff800010122fec: cbz x0, ffff80001012300c : task_faults_idx(): : return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 0.00 : ffff800010122ff0: lsl w1, w22, #1 0.00 : ffff800010122ff4: add w2, w1, #0x1 : group_faults(): : return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 0.00 : ffff800010122ff8: add x1, x0, w1, sxtw #3 : ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 0.00 : ffff800010122ffc: add x0, x0, w2, sxtw #3 : return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 0.00 : ffff800010123000: ldr x26, [x1, #64] 0.00 : ffff800010123004: ldr x0, [x0, #64] 0.00 : ffff800010123008: add x26, x0, x26 : score_nearby_nodes(): : if (sched_numa_topology_type == NUMA_DIRECT) 0.00 : ffff80001012300c: adrp x0, ffff800011afc000 0.00 : ffff800010123010: ldr w0, [x0, #240] 0.00 : ffff800010123014: cbz w0, ffff800010123030 0.00 : ffff800010123018: mov w3, #0x0 // #0 0.00 : ffff80001012301c: mov w2, w25 0.00 : ffff800010123020: mov w1, w22 0.00 : ffff800010123024: mov x0, x20 0.00 : ffff800010123028: bl ffff80001011da50 0.00 : ffff80001012302c: add x26, x26, x0 : group_weight(): : return 1000 * faults / total_faults; 0.00 : ffff800010123030: lsl x0, x26, #5 0.00 : ffff800010123034: sub x0, x0, x26 0.00 : ffff800010123038: add x0, x26, x0, lsl #2 0.00 : ffff80001012303c: ldr x26, [x29, #72] 0.00 : ffff800010123040: lsl x0, x0, #3 0.00 : ffff800010123044: udiv x24, x0, x24 : __read_once_size(): 0.00 : ffff800010123048: ldr x0, [x20, #2248] : group_weight(): : return 0; 0.00 : ffff80001012304c: mov x22, #0x0 // #0 : if (!ng) 0.00 : ffff800010123050: cbz x0, ffff800010123160 : total_faults = ng->total_faults; 0.00 : ffff800010123054: ldr x22, [x0, #40] : if (!total_faults) 0.00 : ffff800010123058: cbz x22, ffff800010123160 0.00 : ffff80001012305c: str x26, [x29, #72] : group_faults(): : return 0; 0.00 : ffff800010123060: mov x26, #0x0 // #0 : __read_once_size(): 0.00 : ffff800010123064: ldr x0, [x20, #2248] : group_faults(): : if (!ng) 0.00 : ffff800010123068: cbz x0, ffff800010123088 : task_faults_idx(): : return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 0.00 : ffff80001012306c: lsl w1, w23, #1 0.00 : ffff800010123070: add w2, w1, #0x1 : group_faults(): : return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 0.00 : ffff800010123074: add x1, x0, w1, sxtw #3 : ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 0.00 : ffff800010123078: add x0, x0, w2, sxtw #3 : return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 0.00 : ffff80001012307c: ldr x26, [x1, #64] 0.00 : ffff800010123080: ldr x0, [x0, #64] 0.00 : ffff800010123084: add x26, x0, x26 : score_nearby_nodes(): : if (sched_numa_topology_type == NUMA_DIRECT) 0.00 : ffff800010123088: adrp x0, ffff800011afc000 0.00 : ffff80001012308c: ldr w0, [x0, #240] 0.00 : ffff800010123090: cbz w0, ffff8000101230ac 0.00 : ffff800010123094: mov w3, #0x0 // #0 0.00 : ffff800010123098: mov w2, w25 0.00 : ffff80001012309c: mov w1, w23 0.00 : ffff8000101230a0: mov x0, x20 0.00 : ffff8000101230a4: bl ffff80001011da50 0.00 : ffff8000101230a8: add x26, x26, x0 : group_weight(): : return 1000 * faults / total_faults; 0.00 : ffff8000101230ac: lsl x0, x26, #5 0.00 : ffff8000101230b0: sub x0, x0, x26 0.00 : ffff8000101230b4: add x0, x26, x0, lsl #2 0.00 : ffff8000101230b8: ldr x26, [x29, #72] 0.00 : ffff8000101230bc: lsl x0, x0, #3 0.00 : ffff8000101230c0: udiv x22, x0, x22 0.00 : ffff8000101230c4: b ffff800010123160 0.00 : ffff8000101230c8: ldp x22, x23, [x29, #40] 0.00 : ffff8000101230cc: nop 0.00 : ffff8000101230d0: ldr x0, [x19, #8] : task_hot(): : if (p->sched_class != &fair_sched_class) 0.00 : ffff8000101230d4: ldr x2, [x20, #128] 0.00 : ffff8000101230d8: adrp x1, ffff800010d06000 0.00 : ffff8000101230dc: add x1, x1, #0x128 0.00 : ffff8000101230e0: cmp x2, x1 0.00 : ffff8000101230e4: b.eq ffff80001012317c // b.none 0.00 : ffff8000101230e8: ldr x24, [x29, #56] 0.00 : ffff8000101230ec: b ffff800010122f24 : can_migrate_task(): : env->flags |= LBF_DST_PINNED; 0.00 : ffff8000101230f0: ldr w1, [x19, #64] : env->new_dst_cpu = cpu; 0.00 : ffff8000101230f4: str w0, [x19, #40] : env->flags |= LBF_DST_PINNED; 0.00 : ffff8000101230f8: orr w0, w1, #0x4 0.00 : ffff8000101230fc: str w0, [x19, #64] : break; 0.00 : ffff800010123100: ldr x22, [x29, #40] 0.00 : ffff800010123104: b ffff800010122f24 : migrate_degrades_locality(): : if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 0.00 : ffff800010123108: ldr x0, [x19, #8] 0.00 : ffff80001012310c: ldp x22, x23, [x29, #40] 0.00 : ffff800010123110: ldr w2, [x0, #4] 0.00 : ffff800010123114: ldr w1, [x0, #12] 0.00 : ffff800010123118: cmp w2, w1 0.00 : ffff80001012311c: b.ls ffff8000101230d4 // b.plast : can_migrate_task(): : if (tsk_cache_hot <= 0 || 0.00 : ffff800010123120: ldr w0, [x3, #48] 0.00 : ffff800010123124: ldr w1, [x3, #76] 0.00 : ffff800010123128: ldr x24, [x29, #56] 0.00 : ffff80001012312c: cmp w1, w0 0.00 : ffff800010123130: cset w21, hi // hi = pmore 0.00 : ffff800010123134: b ffff800010122f24 : migrate_degrades_locality(): : src_weight = task_weight(p, src_nid, dist); 0.00 : ffff800010123138: mov w2, w0 0.00 : ffff80001012313c: mov w1, w22 0.00 : ffff800010123140: mov x0, x20 0.00 : ffff800010123144: bl ffff80001011dc70 : dst_weight = task_weight(p, dst_nid, dist); 0.00 : ffff800010123148: mov w2, w25 : src_weight = task_weight(p, src_nid, dist); 0.00 : ffff80001012314c: mov x24, x0 : dst_weight = task_weight(p, dst_nid, dist); 0.00 : ffff800010123150: mov w1, w23 0.00 : ffff800010123154: mov x0, x20 0.00 : ffff800010123158: bl ffff80001011dc70 0.00 : ffff80001012315c: mov x22, x0 : return dst_weight < src_weight; 0.00 : ffff800010123160: cmp x24, x22 0.00 : ffff800010123164: ldr x25, [x29, #64] 0.00 : ffff800010123168: cset w0, hi // hi = pmore 0.00 : ffff80001012316c: ldp x22, x23, [x29, #40] : can_migrate_task(): : if (tsk_cache_hot <= 0 || 0.00 : ffff800010123170: cbz w0, ffff8000101230e8 0.00 : ffff800010123174: ldr x3, [x19] 0.00 : ffff800010123178: b ffff800010123120 : task_hot(): : if (unlikely(task_has_idle_policy(p))) 0.00 : ffff80001012317c: ldr w1, [x20, #728] 0.00 : ffff800010123180: cmp w1, #0x5 0.00 : ffff800010123184: b.eq ffff8000101230e8 // b.none : if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 0.00 : ffff800010123188: ldr x1, [x19, #24] 0.00 : ffff80001012318c: ldr w1, [x1, #4] 0.00 : ffff800010123190: cbz w1, ffff8000101231b4 : (&p->se == cfs_rq_of(&p->se)->next || 0.00 : ffff800010123194: ldr x2, [x20, #320] 0.00 : ffff800010123198: add x1, x20, #0xc0 : if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 0.00 : ffff80001012319c: ldr x3, [x2, #80] 0.00 : ffff8000101231a0: cmp x1, x3 0.00 : ffff8000101231a4: b.eq ffff800010123174 // b.none : (&p->se == cfs_rq_of(&p->se)->next || 0.00 : ffff8000101231a8: ldr x2, [x2, #88] 0.00 : ffff8000101231ac: cmp x1, x2 0.00 : ffff8000101231b0: b.eq ffff800010123174 // b.none : delta = rq_clock_task(env->src_rq) - p->se.exec_start; 0.00 : ffff8000101231b4: ldr x0, [x0, #2432] : return delta < (s64)sysctl_sched_migration_cost; 0.00 : ffff8000101231b8: mov x1, #0xa11f // #41247 : delta = rq_clock_task(env->src_rq) - p->se.exec_start; 0.00 : ffff8000101231bc: ldr x2, [x20, #264] : return delta < (s64)sysctl_sched_migration_cost; 0.00 : ffff8000101231c0: movk x1, #0x7, lsl #16 : delta = rq_clock_task(env->src_rq) - p->se.exec_start; 0.00 : ffff8000101231c4: sub x0, x0, x2 : return delta < (s64)sysctl_sched_migration_cost; 0.00 : ffff8000101231c8: cmp x0, x1 0.00 : ffff8000101231cc: cset w0, le : can_migrate_task(): : if (tsk_cache_hot <= 0 || 0.00 : ffff8000101231d0: cbnz w0, ffff800010123174 0.00 : ffff8000101231d4: b ffff8000101230e8 0.00 : ffff8000101231d8: ldp x22, x23, [x29, #40] 0.00 : ffff8000101231dc: ldr x24, [x29, #56] 0.00 : ffff8000101231e0: b ffff800010122f24 Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001053cdc0 : : pci_pme_list_scan(): : return !!(dev->pme_support & (1 << state)); : } : EXPORT_SYMBOL(pci_pme_capable); : : static void pci_pme_list_scan(struct work_struct *work) : { 0.00 : ffff80001053cdc0: stp x29, x30, [sp, #-64]! 0.00 : ffff80001053cdc4: mov x29, sp 0.00 : ffff80001053cdc8: stp x20, x21, [sp, #24] 0.00 : ffff80001053cdcc: str x22, [sp, #40] : struct pci_pme_device *pme_dev, *n; : : mutex_lock(&pci_pme_list_mutex); 0.00 : ffff80001053cdd0: adrp x22, ffff80001199d000 0.00 : ffff80001053cdd4: add x21, x22, #0x5c0 0.00 : ffff80001053cdd8: add x0, x21, #0x88 0.00 : ffff80001053cddc: bl ffff800010cd4b28 : list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) { 0.00 : ffff80001053cde0: ldr x1, [x21, #168]! 0.00 : ffff80001053cde4: cmp x1, x21 0.00 : ffff80001053cde8: ldr x20, [x1] 0.00 : ffff80001053cdec: b.eq ffff80001053ce88 // b.none 0.00 : ffff80001053cdf0: stp x23, x24, [x29, #48] : list_del(): : * in an undefined state. : */ : static inline void list_del(struct list_head *entry) : { : __list_del_entry(entry); : entry->next = LIST_POISON1; 0.00 : ffff80001053cdf4: mov x24, #0x100 // #256 : entry->prev = LIST_POISON2; 0.00 : ffff80001053cdf8: mov x23, #0x122 // #290 : entry->next = LIST_POISON1; 0.00 : ffff80001053cdfc: movk x24, #0xdead, lsl #48 : entry->prev = LIST_POISON2; 0.00 : ffff80001053ce00: movk x23, #0xdead, lsl #48 0.00 : ffff80001053ce04: str x19, [x29, #16] 0.00 : ffff80001053ce08: b ffff80001053ce4c : pci_pme_list_scan(): : if (pme_dev->dev->pme_poll) { : struct pci_dev *bridge; : : bridge = pme_dev->dev->bus->self; 100.00 : ffff80001053ce0c: ldr x1, [x19, #16] : pci_pme_wakeup(): : if (pci_check_pme_status(dev)) { 0.00 : ffff80001053ce10: mov x0, x19 : pci_pme_list_scan(): : bridge = pme_dev->dev->bus->self; 0.00 : ffff80001053ce14: ldr x1, [x1, #56] : /* : * If bridge is in low power state, the : * configuration space of subordinate devices : * may be not accessible : */ : if (bridge && bridge->current_state != PCI_D0) 0.00 : ffff80001053ce18: cbz x1, ffff80001053ce24 0.00 : ffff80001053ce1c: ldr w1, [x1, #136] 0.00 : ffff80001053ce20: cbnz w1, ffff80001053ce3c : continue; : /* : * If the device is in D3cold it should not be : * polled either. : */ : if (pme_dev->dev->current_state == PCI_D3cold) 0.00 : ffff80001053ce24: ldr w1, [x19, #136] 0.00 : ffff80001053ce28: cmp w1, #0x4 0.00 : ffff80001053ce2c: b.eq ffff80001053ce3c // b.none : pci_pme_wakeup(): : if (pci_check_pme_status(dev)) { 0.00 : ffff80001053ce30: bl ffff80001053cca0 0.00 : ffff80001053ce34: tst w0, #0xff 0.00 : ffff80001053ce38: b.ne ffff80001053ced0 // b.any 0.00 : ffff80001053ce3c: mov x1, x20 : pci_pme_list_scan(): : list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) { 0.00 : ffff80001053ce40: cmp x20, x21 0.00 : ffff80001053ce44: ldr x20, [x20] 0.00 : ffff80001053ce48: b.eq ffff80001053ce80 // b.none : if (pme_dev->dev->pme_poll) { 0.00 : ffff80001053ce4c: ldr x19, [x1, #16] 0.00 : ffff80001053ce50: ldrb w0, [x19, #142] 0.00 : ffff80001053ce54: tbnz w0, #5, ffff80001053ce0c : __list_del_entry(): : __list_del(entry->prev, entry->next); 0.00 : ffff80001053ce58: ldp x3, x2, [x1] : __list_del(): : next->prev = prev; 0.00 : ffff80001053ce5c: str x2, [x3, #8] : pci_pme_list_scan(): : continue; : : pci_pme_wakeup(pme_dev->dev, NULL); : } else { : list_del(&pme_dev->list); : kfree(pme_dev); 0.00 : ffff80001053ce60: mov x0, x1 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001053ce64: str x3, [x2] : list_del(): : entry->prev = LIST_POISON2; 0.00 : ffff80001053ce68: stp x24, x23, [x1] : pci_pme_list_scan(): 0.00 : ffff80001053ce6c: bl ffff800010253890 0.00 : ffff80001053ce70: mov x1, x20 : list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) { 0.00 : ffff80001053ce74: cmp x20, x21 0.00 : ffff80001053ce78: ldr x20, [x20] 0.00 : ffff80001053ce7c: b.ne ffff80001053ce4c // b.any 0.00 : ffff80001053ce80: ldr x19, [x29, #16] 0.00 : ffff80001053ce84: ldp x23, x24, [x29, #48] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001053ce88: add x2, x22, #0x5c0 0.00 : ffff80001053ce8c: mov x0, x2 0.00 : ffff80001053ce90: ldr x1, [x0, #168]! : pci_pme_list_scan(): : } : } : if (!list_empty(&pci_pme_list)) 0.00 : ffff80001053ce94: cmp x1, x0 0.00 : ffff80001053ce98: b.eq ffff80001053ceb4 // b.none : queue_delayed_work(system_freezable_wq, &pci_pme_work, 0.00 : ffff80001053ce9c: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : queue_delayed_work(): : */ : static inline bool queue_delayed_work(struct workqueue_struct *wq, : struct delayed_work *dwork, : unsigned long delay) : { : return queue_delayed_work_on(WORK_CPU_UNBOUND, wq, dwork, delay); 0.00 : ffff80001053cea0: mov x3, #0xfa // #250 0.00 : ffff80001053cea4: add x2, x2, #0xb8 0.00 : ffff80001053cea8: mov w0, #0x100 // #256 0.00 : ffff80001053ceac: ldr x1, [x1, #528] 0.00 : ffff80001053ceb0: bl ffff800010105988 : pci_pme_list_scan(): : msecs_to_jiffies(PME_TIMEOUT)); : mutex_unlock(&pci_pme_list_mutex); 0.00 : ffff80001053ceb4: add x0, x22, #0x5c0 0.00 : ffff80001053ceb8: add x0, x0, #0x88 0.00 : ffff80001053cebc: bl ffff800010cd4428 : } 0.00 : ffff80001053cec0: ldp x20, x21, [sp, #24] 0.00 : ffff80001053cec4: ldr x22, [sp, #40] 0.00 : ffff80001053cec8: ldp x29, x30, [sp], #64 0.00 : ffff80001053cecc: ret : pci_wakeup_event(): : void pci_bridge_wait_for_secondary_bus(struct pci_dev *dev); : : static inline void pci_wakeup_event(struct pci_dev *dev) : { : /* Wait 100 ms before the system can be put into a sleep state. */ : pm_wakeup_event(&dev->dev, 100); 0.00 : ffff80001053ced0: add x19, x19, #0xb0 : pm_wakeup_event(): : return pm_wakeup_ws_event(ws, msec, false); : } : : static inline void pm_wakeup_event(struct device *dev, unsigned int msec) : { : return pm_wakeup_dev_event(dev, msec, false); 0.00 : ffff80001053ced4: mov w2, #0x0 // #0 0.00 : ffff80001053ced8: mov x0, x19 0.00 : ffff80001053cedc: mov w1, #0x64 // #100 0.00 : ffff80001053cee0: bl ffff8000107459c0 : pm_request_resume(): : return __pm_runtime_idle(dev, RPM_ASYNC); : } : : static inline int pm_request_resume(struct device *dev) : { : return __pm_runtime_resume(dev, RPM_ASYNC); 0.00 : ffff80001053cee4: mov w1, #0x1 // #1 0.00 : ffff80001053cee8: mov x0, x19 0.00 : ffff80001053ceec: bl ffff80001073fa90 <__pm_runtime_resume> 0.00 : ffff80001053cef0: b ffff80001053ce3c Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015cda8 : : rcu_report_qs_rnp(): : * during grace-period initialization. : */ : static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, : unsigned long gps, unsigned long flags) : __releases(rnp->lock) : { 0.00 : ffff80001015cda8: stp x29, x30, [sp, #-64]! 0.00 : ffff80001015cdac: mov x29, sp 0.00 : ffff80001015cdb0: stp x19, x20, [sp, #16] 0.00 : ffff80001015cdb4: mov x19, x1 0.00 : ffff80001015cdb8: mov x20, x0 0.00 : ffff80001015cdbc: mov x1, x3 : : raw_lockdep_assert_held_rcu_node(rnp); : : /* Walk up the rcu_node hierarchy. */ : for (;;) { : if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) { 0.00 : ffff80001015cdc0: ldr x0, [x19, #32] 0.00 : ffff80001015cdc4: tst x20, x0 0.00 : ffff80001015cdc8: ccmp x20, #0x0, #0x4, eq // eq = none 0.00 : ffff80001015cdcc: b.ne ffff80001015ceec // b.any 0.00 : ffff80001015cdd0: str x22, [x29, #40] 0.00 : ffff80001015cdd4: mov x22, x2 0.00 : ffff80001015cdd8: ldr x0, [x19, #8] 0.00 : ffff80001015cddc: cmp x2, x0 0.00 : ffff80001015cde0: b.ne ffff80001015cee8 // b.any 0.00 : ffff80001015cde4: str x23, [x29, #48] 0.00 : ffff80001015cde8: adrp x23, ffff80001190a000 <__per_cpu_offset+0x6d8> : */ : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); : return; : } : WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */ : WARN_ON_ONCE(!rcu_is_leaf_node(rnp) && 0.00 : ffff80001015cdec: add x23, x23, #0x2a0 0.00 : ffff80001015cdf0: str x21, [x29, #32] 0.00 : ffff80001015cdf4: nop 0.00 : ffff80001015cdf8: ldr w2, [x23, #4] 0.00 : ffff80001015cdfc: ldrb w0, [x19, #113] 0.00 : ffff80001015ce00: sub w2, w2, #0x1 0.00 : ffff80001015ce04: cmp w0, w2 0.00 : ffff80001015ce08: b.eq ffff80001015ce18 // b.none : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001015ce0c: ldr x0, [x19, #144] : rcu_report_qs_rnp(): 0.00 : ffff80001015ce10: cbz x0, ffff80001015ce18 0.00 : ffff80001015ce14: brk #0x800 : rcu_preempt_blocked_readers_cgp(rnp)); : rnp->qsmask &= ~mask; 0.00 : ffff80001015ce18: ldr x0, [x19, #32] : trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq, 0.00 : ffff80001015ce1c: ldr x2, [x19, #8] : rnp->qsmask &= ~mask; 0.00 : ffff80001015ce20: bic x20, x0, x20 0.00 : ffff80001015ce24: str x20, [x19, #32] : mask, rnp->qsmask, rnp->level, : rnp->grplo, rnp->grphi, : !!rnp->gp_tasks); : if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) { 0.00 : ffff80001015ce28: cbnz x20, ffff80001015ce98 : __read_once_size(): 0.00 : ffff80001015ce2c: ldr x0, [x19, #144] : rcu_report_qs_rnp(): 0.00 : ffff80001015ce30: cbnz x0, ffff80001015ce98 : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); : return; : } : rnp->completedqs = rnp->gp_seq; : mask = rnp->grpmask; : if (rnp->parent == NULL) { 0.00 : ffff80001015ce34: ldr x0, [x19, #120] : rnp->completedqs = rnp->gp_seq; 0.00 : ffff80001015ce38: str x2, [x19, #24] : mask = rnp->grpmask; 0.00 : ffff80001015ce3c: ldr x20, [x19, #96] : if (rnp->parent == NULL) { 0.00 : ffff80001015ce40: cbz x0, ffff80001015cecc : : /* No more levels. Exit loop holding root lock. */ : : break; : } : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff80001015ce44: mov x0, x19 0.00 : ffff80001015ce48: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : rnp_c = rnp; : rnp = rnp->parent; 0.00 : ffff80001015ce4c: ldr x21, [x19, #120] : raw_spin_lock_irqsave_rcu_node(rnp, flags); 0.00 : ffff80001015ce50: mov x0, x21 0.00 : ffff80001015ce54: bl ffff800010cd8640 <_raw_spin_lock_irqsave> : if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) { 0.00 : ffff80001015ce58: ldr x2, [x21, #32] : raw_spin_lock_irqsave_rcu_node(rnp, flags); 0.00 : ffff80001015ce5c: mov x1, x0 : oldmask = rnp_c->qsmask; 0.00 : ffff80001015ce60: ldr x3, [x19, #32] : raw_spin_lock_irqsave_rcu_node(rnp, flags); 0.00 : ffff80001015ce64: mov x0, x21 : if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) { 0.00 : ffff80001015ce68: tst x20, x2 0.00 : ffff80001015ce6c: ccmp x20, #0x0, #0x4, eq // eq = none 100.00 : ffff80001015ce70: b.ne ffff80001015ceb4 // b.any 0.00 : ffff80001015ce74: ldr x2, [x21, #8] 0.00 : ffff80001015ce78: cmp x2, x22 0.00 : ffff80001015ce7c: b.ne ffff80001015ceb4 // b.any : WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */ 0.00 : ffff80001015ce80: cbnz x3, ffff80001015ce8c : rnp = rnp->parent; 0.00 : ffff80001015ce84: mov x19, x21 0.00 : ffff80001015ce88: b ffff80001015cdf8 : WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */ 0.00 : ffff80001015ce8c: brk #0x800 : rnp = rnp->parent; 0.00 : ffff80001015ce90: mov x19, x21 0.00 : ffff80001015ce94: b ffff80001015cdf8 : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff80001015ce98: mov x0, x19 0.00 : ffff80001015ce9c: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : return; 0.00 : ffff80001015cea0: ldp x21, x22, [x29, #32] 0.00 : ffff80001015cea4: ldr x23, [x29, #48] : * Get here if we are the last CPU to pass through a quiescent : * state for this grace period. Invoke rcu_report_qs_rsp() : * to clean up and start the next grace period if one is needed. : */ : rcu_report_qs_rsp(flags); /* releases rnp->lock. */ : } 0.00 : ffff80001015cea8: ldp x19, x20, [sp, #16] 0.00 : ffff80001015ceac: ldp x29, x30, [sp], #64 0.00 : ffff80001015ceb0: ret 0.00 : ffff80001015ceb4: ldp x21, x22, [x29, #32] 0.00 : ffff80001015ceb8: ldr x23, [x29, #48] : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff80001015cebc: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : } 0.00 : ffff80001015cec0: ldp x19, x20, [sp, #16] 0.00 : ffff80001015cec4: ldp x29, x30, [sp], #64 0.00 : ffff80001015cec8: ret : rcu_report_qs_rsp(flags); /* releases rnp->lock. */ 0.00 : ffff80001015cecc: mov x0, x1 0.00 : ffff80001015ced0: bl ffff80001015cd58 0.00 : ffff80001015ced4: ldp x21, x22, [x29, #32] 0.00 : ffff80001015ced8: ldr x23, [x29, #48] : } 0.00 : ffff80001015cedc: ldp x19, x20, [sp, #16] 0.00 : ffff80001015cee0: ldp x29, x30, [sp], #64 0.00 : ffff80001015cee4: ret 0.00 : ffff80001015cee8: ldr x22, [x29, #40] 0.00 : ffff80001015ceec: mov x0, x19 : raw_spin_unlock_irqrestore_rcu_node(rnp, flags); 0.00 : ffff80001015cef0: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> 0.00 : ffff80001015cef4: b ffff80001015cec0 Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101306e8 : : swake_up_one(): : list_del_init(&curr->task_list); : } : EXPORT_SYMBOL(swake_up_locked); : : void swake_up_one(struct swait_queue_head *q) : { 0.00 : ffff8000101306e8: stp x29, x30, [sp, #-32]! 0.00 : ffff8000101306ec: mov x29, sp 100.00 : ffff8000101306f0: stp x19, x20, [sp, #16] 0.00 : ffff8000101306f4: mov x19, x0 : unsigned long flags; : : raw_spin_lock_irqsave(&q->lock, flags); 0.00 : ffff8000101306f8: bl ffff800010cd8640 <_raw_spin_lock_irqsave> 0.00 : ffff8000101306fc: mov x20, x0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010130700: mov x0, x19 0.00 : ffff800010130704: ldr x1, [x0, #8]! : swake_up_locked(): : if (list_empty(&q->task_list)) 0.00 : ffff800010130708: cmp x1, x0 0.00 : ffff80001013070c: b.eq ffff800010130718 // b.none 0.00 : ffff800010130710: mov x0, x19 0.00 : ffff800010130714: bl ffff800010130688 : swake_up_one(): : swake_up_locked(q); : raw_spin_unlock_irqrestore(&q->lock, flags); 0.00 : ffff800010130718: mov x1, x20 0.00 : ffff80001013071c: mov x0, x19 0.00 : ffff800010130720: bl ffff800010cd8540 <_raw_spin_unlock_irqrestore> : } 0.00 : ffff800010130724: ldp x19, x20, [sp, #16] 0.00 : ffff800010130728: ldp x29, x30, [sp], #32 0.00 : ffff80001013072c: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010102338 : : set_work_pool_and_clear_pending(): : * The following wmb is paired with the implied mb in : * test_and_set_bit(PENDING) and ensures all updates to @work made : * here are visible to and precede any updates by the next PENDING : * owner. : */ : smp_wmb(); 0.00 : ffff800010102338: dmb ishst : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 100.00 : ffff80001010233c: ldr x2, [x0] : set_work_pool_and_clear_pending(): : set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0); 0.00 : ffff800010102340: sbfiz x1, x1, #5, #32 : set_work_data(): : WARN_ON_ONCE(!work_pending(work)); 0.00 : ffff800010102344: tbz w2, #0, ffff800010102354 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010102348: str x1, [x0] : set_work_pool_and_clear_pending(): : * a @work is not queued in a hope, that CPU#1 will eventually : * finish the queued @work. Meanwhile CPU#1 does not see : * event_indicated is set, because speculative LOAD was executed : * before actual STORE. : */ : smp_mb(); 0.00 : ffff80001010234c: dmb ish : } 0.00 : ffff800010102350: ret : set_work_data(): : WARN_ON_ONCE(!work_pending(work)); 0.00 : ffff800010102354: brk #0x800 : __write_once_size(): 0.00 : ffff800010102358: str x1, [x0] : set_work_pool_and_clear_pending(): : smp_mb(); 0.00 : ffff80001010235c: dmb ish : } 0.00 : ffff800010102360: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd2de0 : : preempt_schedule_common(): : schedule(); : preempt_disable(); : } : : static void __sched notrace preempt_schedule_common(void) : { 100.00 : ffff800010cd2de0: stp x29, x30, [sp, #-32]! 0.00 : ffff800010cd2de4: mov x29, sp 0.00 : ffff800010cd2de8: str x19, [sp, #16] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010cd2dec: mrs x19, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010cd2df0: ldr w0, [x19, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff800010cd2df4: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010cd2df8: str w0, [x19, #16] : preempt_schedule_common(): : * traced. The other to still record the preemption latency, : * which can also be traced by the function tracer. : */ : preempt_disable_notrace(); : preempt_latency_start(1); : __schedule(true); 0.00 : ffff800010cd2dfc: mov w0, #0x1 // #1 0.00 : ffff800010cd2e00: bl ffff800010cd24e0 <__schedule> : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010cd2e04: ldr w0, [x19, #16] : __preempt_count_sub(): : } : : static inline void __preempt_count_sub(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc -= val; 0.00 : ffff800010cd2e08: sub w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010cd2e0c: str w0, [x19, #16] : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010cd2e10: ldr x0, [x19] : preempt_schedule_common(): : : /* : * Check again in case we missed a preemption opportunity : * between schedule and now. : */ : } while (need_resched()); 0.00 : ffff800010cd2e14: tbnz w0, #1, ffff800010cd2dec : } 0.00 : ffff800010cd2e18: ldr x19, [sp, #16] 0.00 : ffff800010cd2e1c: ldp x29, x30, [sp], #32 0.00 : ffff800010cd2e20: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010162bb0 : : rcu_segcblist_pend_cbs(): : * Does the specified rcu_segcblist structure contain callbacks that : * are still pending, that is, not yet ready to be invoked? : */ : bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp) : { : return rcu_segcblist_is_enabled(rsclp) && 0.00 : ffff800010162bb0: ldrb w1, [x0, #80] 0.00 : ffff800010162bb4: cbz w1, ffff800010162bcc : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 100.00 : ffff800010162bb8: ldr x0, [x0, #8] 0.00 : ffff800010162bbc: ldr x0, [x0] : rcu_segcblist_pend_cbs(): 0.00 : ffff800010162bc0: cmp x0, #0x0 0.00 : ffff800010162bc4: cset w0, ne // ne = any : !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL); : } 0.00 : ffff800010162bc8: ret 0.00 : ffff800010162bcc: mov w0, #0x0 // #0 0.00 : ffff800010162bd0: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100a53e8 : : pmd_huge(): : } : #endif : : int pmd_huge(pmd_t pmd) : { : return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); 0.00 : ffff8000100a53e8: cmp x0, #0x0 0.00 : ffff8000100a53ec: lsr x0, x0, #1 0.00 : ffff8000100a53f0: eor x0, x0, #0x1 0.00 : ffff8000100a53f4: and w0, w0, #0x1 : } 100.00 : ffff8000100a53f8: csel w0, w0, wzr, ne // ne = any 0.00 : ffff8000100a53fc: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010464600 : : hctx_unlock(): : put_cpu(); : } : : static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx) : __releases(hctx->srcu) : { 49.84 : ffff800010464600: stp x29, x30, [sp, #-16]! 0.00 : ffff800010464604: mov x29, sp : if (!(hctx->flags & BLK_MQ_F_BLOCKING)) 50.16 : ffff800010464608: ldr x2, [x0, #192] 0.00 : ffff80001046460c: tbz w2, #5, ffff800010464628 : rcu_read_unlock(); : else : srcu_read_unlock(hctx->srcu, srcu_idx); 0.00 : ffff800010464610: add x0, x0, #0x240 : srcu_read_unlock(): : * Exit an SRCU read-side critical section. : */ : static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) : __releases(ssp) : { : WARN_ON_ONCE(idx & ~0x1); 0.00 : ffff800010464614: tst w1, #0xfffffffe 0.00 : ffff800010464618: b.ne ffff800010464634 // b.any : rcu_lock_release(&(ssp)->dep_map); : __srcu_read_unlock(ssp, idx); 0.00 : ffff80001046461c: bl ffff80001015b368 <__srcu_read_unlock> : hctx_unlock(): : } 0.00 : ffff800010464620: ldp x29, x30, [sp], #16 0.00 : ffff800010464624: ret : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff800010464628: bl ffff800010160d20 <__rcu_read_unlock> : hctx_unlock(): 0.00 : ffff80001046462c: ldp x29, x30, [sp], #16 0.00 : ffff800010464630: ret : srcu_read_unlock(): : WARN_ON_ONCE(idx & ~0x1); 0.00 : ffff800010464634: brk #0x800 0.00 : ffff800010464638: b ffff80001046461c Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101216b0 : : task_tick_fair(): : * goes along full dynticks. Therefore no local assumption can be made : * and everything must be accessed through the @rq and @curr passed in : * parameters. : */ : static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) : { 0.00 : ffff8000101216b0: stp x29, x30, [sp, #-128]! 0.00 : ffff8000101216b4: mov x29, sp 0.00 : ffff8000101216b8: str x20, [sp, #24] : struct cfs_rq *cfs_rq; : struct sched_entity *se = &curr->se; : : for_each_sched_entity(se) { 0.00 : ffff8000101216bc: adds x20, x1, #0xc0 : { 0.00 : ffff8000101216c0: stp x23, x24, [sp, #48] 0.00 : ffff8000101216c4: mov x23, x1 0.00 : ffff8000101216c8: str x0, [x29, #96] : for_each_sched_entity(se) { 0.00 : ffff8000101216cc: b.eq ffff800010121c78 // b.none 0.00 : ffff8000101216d0: stp x25, x26, [x29, #64] : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff8000101216d4: adrp x25, ffff800011afb000 0.00 : ffff8000101216d8: add x0, x25, #0xd80 0.00 : ffff8000101216dc: stp x21, x22, [x29, #32] : cpufreq_update_util(): : */ : static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) : { : struct update_util_data *data; : : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff8000101216e0: adrp x26, ffff80001151f000 : update_tg_load_avg(): 0.00 : ffff8000101216e4: str x0, [x29, #120] 0.00 : ffff8000101216e8: mov w22, w2 : cpufreq_update_util(): 0.00 : ffff8000101216ec: add x0, x26, #0x158 0.00 : ffff8000101216f0: str x19, [x29, #16] 0.00 : ffff8000101216f4: stp x27, x28, [x29, #80] 0.00 : ffff8000101216f8: str x0, [x29, #104] 0.00 : ffff8000101216fc: adrp x0, ffff800011909000 0.00 : ffff800010121700: str x0, [x29, #112] 0.00 : ffff800010121704: nop : task_tick_fair(): : cfs_rq = cfs_rq_of(se); 0.00 : ffff800010121708: ldr x19, [x20, #128] : entity_tick(): : update_curr(cfs_rq); 0.00 : ffff80001012170c: mov x0, x19 0.00 : ffff800010121710: bl ffff80001011fab8 : cfs_rq_clock_pelt(): : return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; : } : #else : static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) : { : return rq_clock_pelt(rq_of(cfs_rq)); 0.00 : ffff800010121714: ldr x0, [x19, #304] : update_load_avg(): : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 0.00 : ffff800010121718: ldr x1, [x20, #192] : rq_clock_pelt(): : return rq->clock_pelt - rq->lost_idle_time; 0.00 : ffff80001012171c: ldr x3, [x0, #2440] 0.00 : ffff800010121720: ldr x0, [x0, #2448] 0.00 : ffff800010121724: sub x28, x3, x0 : update_load_avg(): 0.00 : ffff800010121728: cbz x1, ffff80001012173c : __update_load_avg_se(now, cfs_rq, se); 0.00 : ffff80001012172c: mov x2, x20 0.00 : ffff800010121730: mov x1, x19 0.00 : ffff800010121734: mov x0, x28 0.00 : ffff800010121738: bl ffff800010134868 <__update_load_avg_se> : update_cfs_rq_load_avg(): : if (cfs_rq->removed.nr) { 0.00 : ffff80001012173c: ldr w21, [x19, #196] 0.00 : ffff800010121740: cbz w21, ffff8000101217e4 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010121744: ldr w1, [x19, #156] 0.00 : ffff800010121748: mov w2, #0xb67e // #46718 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff80001012174c: add x21, x19, #0xc0 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010121750: add w24, w1, w2 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010121754: mov x0, x21 0.00 : ffff800010121758: bl ffff800010cd80f0 <_raw_spin_lock> : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff80001012175c: ldr x25, [x19, #216] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff800010121760: ldp x27, x26, [x19, #200] : cfs_rq->removed.nr = 0; 0.00 : ffff800010121764: str wzr, [x19, #196] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff800010121768: stp xzr, xzr, [x19, #200] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff80001012176c: mov x0, x21 : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010121770: str xzr, [x19, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff800010121774: bl ffff800010cd7c38 <_raw_spin_unlock> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010121778: ldr x7, [x19, #160] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff80001012177c: mov w0, w24 : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010121780: ldr x4, [x19, #272] : cfs_rq->propagate = 1; 0.00 : ffff800010121784: mov x6, #0x1 // #1 : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff800010121788: sub x2, x7, x27 : decayed = 1; 0.00 : ffff80001012178c: mov w21, w6 : sub_positive(&sa->load_avg, r); 0.00 : ffff800010121790: cmp x7, x2 : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010121794: str x6, [x19, #264] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff800010121798: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012179c: str x2, [x19, #160] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff8000101217a0: sub x2, x4, x25 0.00 : ffff8000101217a4: str x2, [x19, #272] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101217a8: ldr x4, [x19, #136] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff8000101217ac: msub x0, x0, x27, x4 0.00 : ffff8000101217b0: cmp x4, x0 0.00 : ffff8000101217b4: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101217b8: str x0, [x19, #136] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101217bc: ldr x2, [x19, #176] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_avg, r); 0.00 : ffff8000101217c0: sub x0, x2, x26 0.00 : ffff8000101217c4: cmp x2, x0 0.00 : ffff8000101217c8: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101217cc: str x0, [x19, #176] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101217d0: ldr w2, [x19, #152] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_sum, r * divider); 0.00 : ffff8000101217d4: msub w0, w24, w26, w2 0.00 : ffff8000101217d8: cmp w2, w0 0.00 : ffff8000101217dc: csel w0, w0, wzr, cs // cs = hs, nlast : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101217e0: str w0, [x19, #152] : update_cfs_rq_load_avg(): : decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 0.00 : ffff8000101217e4: mov x1, x19 0.00 : ffff8000101217e8: mov x0, x28 0.00 : ffff8000101217ec: bl ffff800010134a68 <__update_load_avg_cfs_rq> 0.00 : ffff8000101217f0: orr w0, w0, w21 : propagate_entity_load_avg(): : if (entity_is_task(se)) 0.00 : ffff8000101217f4: ldr x1, [x20, #136] 0.00 : ffff8000101217f8: cbz x1, ffff800010121af4 : if (!gcfs_rq->propagate) 0.00 : ffff8000101217fc: ldr x2, [x1, #264] 0.00 : ffff800010121800: cbz x2, ffff800010121af4 : gcfs_rq->propagate = 0; 0.00 : ffff800010121804: str xzr, [x1, #264] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010121808: mov x3, #0x1 // #1 : propagate_entity_load_avg(): : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 0.00 : ffff80001012180c: ldr x2, [x1, #272] : cfs_rq = cfs_rq_of(se); 0.00 : ffff800010121810: ldr x4, [x20, #128] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010121814: ldr x0, [x4, #272] 0.00 : ffff800010121818: add x0, x0, x2 0.00 : ffff80001012181c: stp x3, x0, [x4, #264] : propagate_entity_load_avg(): : update_tg_cfs_util(cfs_rq, se, gcfs_rq); 0.00 : ffff800010121820: ldr x2, [x1, #176] : update_tg_cfs_util(): : long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 0.00 : ffff800010121824: ldr x0, [x20, #240] 0.00 : ffff800010121828: sub x0, x2, x0 : if (!delta) 0.00 : ffff80001012182c: cmp x0, #0x0 0.00 : ffff800010121830: b.eq ffff800010121864 // b.none : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010121834: mov w3, #0xba7e // #47742 : se->avg.util_avg = gcfs_rq->avg.util_avg; 0.00 : ffff800010121838: str x2, [x20, #240] : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff80001012183c: mul w2, w2, w3 0.00 : ffff800010121840: str w2, [x20, #216] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121844: ldr x2, [x4, #176] : update_tg_cfs_util(): : add_positive(&cfs_rq->avg.util_avg, delta); 0.00 : ffff800010121848: add x0, x0, x2 0.00 : ffff80001012184c: ccmp x2, x0, #0x2, lt // lt = tstop 0.00 : ffff800010121850: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010121854: str x0, [x4, #176] : update_tg_cfs_util(): : cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010121858: ldr x0, [x4, #176] 0.00 : ffff80001012185c: mul w3, w3, w0 0.00 : ffff800010121860: str w3, [x4, #152] : update_tg_cfs_runnable(): : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 0.00 : ffff800010121864: ldr x3, [x1, #272] : if (!runnable_sum) 0.00 : ffff800010121868: cmp x3, #0x0 0.00 : ffff80001012186c: b.eq ffff800010121988 // b.none : gcfs_rq->prop_runnable_sum = 0; 0.00 : ffff800010121870: str xzr, [x1, #272] : if (runnable_sum >= 0) { 0.00 : ffff800010121874: b.lt ffff800010121c10 // b.tstop : runnable_sum += se->avg.load_sum; 0.00 : ffff800010121878: ldr x0, [x20, #200] : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff80001012187c: mov x1, #0xba7e // #47742 : runnable_sum += se->avg.load_sum; 0.00 : ffff800010121880: add x3, x3, x0 : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff800010121884: cmp x3, x1 0.00 : ffff800010121888: csel x3, x3, x1, le : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff80001012188c: ldr w1, [x20, #216] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010121890: ldr x2, [x20] : update_tg_cfs_runnable(): : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff800010121894: ldr x5, [x20, #224] : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff800010121898: lsr w1, w1, #10 : runnable_sum = max(runnable_sum, running_sum); 0.00 : ffff80001012189c: cmp x1, x3 : se_weight(): 0.00 : ffff8000101218a0: lsr x2, x2, #10 : update_tg_cfs_runnable(): 0.00 : ffff8000101218a4: csel x3, x1, x3, ge // ge = tcont : se->avg.load_sum = runnable_sum; 0.00 : ffff8000101218a8: str x3, [x20, #200] : load_sum = (s64)se_weight(se) * runnable_sum; 0.00 : ffff8000101218ac: mul x1, x2, x3 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 0.00 : ffff8000101218b0: msub x2, x2, x0, x1 : div_s64_rem(): : * Return: sets ``*remainder``, then returns dividend / divisor : */ : static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) : { : *remainder = dividend % divisor; : return dividend / divisor; 0.00 : ffff8000101218b4: lsr x0, x1, #1 0.00 : ffff8000101218b8: mov x1, #0x5041 // #20545 0.00 : ffff8000101218bc: movk x1, #0xafb, lsl #16 0.00 : ffff8000101218c0: movk x1, #0x400, lsl #32 0.00 : ffff8000101218c4: movk x1, #0xafb5, lsl #48 0.00 : ffff8000101218c8: umulh x0, x0, x1 0.00 : ffff8000101218cc: lsr x0, x0, #14 : update_tg_cfs_runnable(): : se->avg.load_avg = load_avg; 0.00 : ffff8000101218d0: str x0, [x20, #224] : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff8000101218d4: sub x0, x0, x5 : div_s64_rem(): 0.00 : ffff8000101218d8: mov x5, #0x5041 // #20545 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101218dc: ldr x1, [x4, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_avg, delta_avg); 0.00 : ffff8000101218e0: cmp x0, #0x0 : div_s64_rem(): 0.00 : ffff8000101218e4: movk x5, #0xafb, lsl #16 : update_tg_cfs_runnable(): 0.00 : ffff8000101218e8: add x0, x0, x1 : div_s64_rem(): 0.00 : ffff8000101218ec: movk x5, #0x400, lsl #32 : update_tg_cfs_runnable(): 0.00 : ffff8000101218f0: ccmp x1, x0, #0x2, lt // lt = tstop : div_s64_rem(): 0.00 : ffff8000101218f4: movk x5, #0xafb5, lsl #48 : update_tg_cfs_runnable(): 0.00 : ffff8000101218f8: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101218fc: str x0, [x4, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_sum, delta_sum); 0.00 : ffff800010121900: cmp x2, #0x0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121904: ldr x1, [x4, #136] : update_tg_cfs_runnable(): 0.00 : ffff800010121908: add x0, x2, x1 0.00 : ffff80001012190c: ccmp x1, x0, #0x2, lt // lt = tstop 0.00 : ffff800010121910: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010121914: str x0, [x4, #136] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff800010121918: ldr x1, [x20, #16] : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff80001012191c: ldr w2, [x20, #64] : se_runnable(): 0.00 : ffff800010121920: lsr x1, x1, #10 : update_tg_cfs_runnable(): : runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 0.00 : ffff800010121924: mul x1, x1, x3 : div_s64_rem(): 0.00 : ffff800010121928: lsr x0, x1, #1 0.00 : ffff80001012192c: umulh x0, x0, x5 0.00 : ffff800010121930: lsr x0, x0, #14 : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff800010121934: cbz w2, ffff800010121980 : delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 0.00 : ffff800010121938: ldr x6, [x20, #232] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012193c: ldr x7, [x4, #168] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010121940: ldr x2, [x20] : update_tg_cfs_runnable(): 0.00 : ffff800010121944: sub x6, x0, x6 : se_weight(se) * se->avg.runnable_load_sum; 0.00 : ffff800010121948: ldr x8, [x20, #208] : add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 0.00 : ffff80001012194c: add x5, x6, x7 0.00 : ffff800010121950: cmp x6, #0x0 0.00 : ffff800010121954: ccmp x7, x5, #0x2, lt // lt = tstop : se_weight(): 0.00 : ffff800010121958: lsr x2, x2, #10 : update_tg_cfs_runnable(): 0.00 : ffff80001012195c: csel x5, x5, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010121960: str x5, [x4, #168] : update_tg_cfs_runnable(): : delta_sum = runnable_load_sum - 0.00 : ffff800010121964: msub x1, x2, x8, x1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121968: ldr x2, [x4, #144] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 0.00 : ffff80001012196c: cmp x1, #0x0 0.00 : ffff800010121970: add x1, x1, x2 0.00 : ffff800010121974: ccmp x2, x1, #0x2, lt // lt = tstop 0.00 : ffff800010121978: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012197c: str x1, [x4, #144] : update_tg_cfs_runnable(): : se->avg.runnable_load_sum = runnable_sum; 0.00 : ffff800010121980: str x3, [x20, #208] : se->avg.runnable_load_avg = runnable_load_avg; 0.00 : ffff800010121984: str x0, [x20, #232] : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010121988: ldr x1, [x19, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff80001012198c: add x0, x1, #0x80 0.00 : ffff800010121990: cmp x19, x0 0.00 : ffff800010121994: b.eq ffff800010121b18 // b.none : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff800010121998: ldr x0, [x19, #336] 0.00 : ffff80001012199c: ldr x1, [x29, #120] 0.00 : ffff8000101219a0: cmp x0, x1 0.00 : ffff8000101219a4: b.eq ffff8000101219d8 // b.none : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 0.00 : ffff8000101219a8: ldr x1, [x19, #160] 0.00 : ffff8000101219ac: ldr x2, [x19, #256] : if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 0.00 : ffff8000101219b0: subs x1, x1, x2 0.00 : ffff8000101219b4: cneg x3, x1, mi // mi = first 0.00 : ffff8000101219b8: cmp x3, x2, lsr #6 0.00 : ffff8000101219bc: b.ls ffff8000101219d8 // b.plast : atomic_long_add(delta, &cfs_rq->tg->load_avg); 0.00 : ffff8000101219c0: add x2, x0, #0x140 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101219c4: b ffff800010121bfc 0.00 : ffff8000101219c8: b ffff800010121bfc : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff8000101219cc: stadd x1, [x2] : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff8000101219d0: ldr x0, [x19, #160] 0.00 : ffff8000101219d4: str x0, [x19, #256] : entity_tick(): : update_cfs_group(curr); 0.00 : ffff8000101219d8: mov x0, x20 0.00 : ffff8000101219dc: bl ffff80001011fd78 : resched_curr(rq_of(cfs_rq)); 0.00 : ffff8000101219e0: ldr x0, [x19, #304] : if (queued) { 0.00 : ffff8000101219e4: cbnz w22, ffff800010121b08 : hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) 0.00 : ffff8000101219e8: add x0, x0, #0xb60 0.00 : ffff8000101219ec: bl ffff80001016aeb8 : if (!sched_feat(DOUBLE_TICK) && 0.00 : ffff8000101219f0: tst w0, #0xff 0.00 : ffff8000101219f4: b.ne ffff800010121a58 // b.any : if (cfs_rq->nr_running > 1) 0.00 : ffff8000101219f8: ldr w0, [x19, #24] 0.00 : ffff8000101219fc: cmp w0, #0x1 0.00 : ffff800010121a00: b.ls ffff800010121a58 // b.plast : check_preempt_tick(): : ideal_runtime = sched_slice(cfs_rq, curr); 0.00 : ffff800010121a04: mov x1, x20 0.00 : ffff800010121a08: bl ffff80001011d0a8 : delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; 0.00 : ffff800010121a0c: ldr x1, [x20, #80] 0.00 : ffff800010121a10: ldr x2, [x20, #96] 0.00 : ffff800010121a14: sub x1, x1, x2 : if (delta_exec > ideal_runtime) { 0.00 : ffff800010121a18: cmp x0, x1 0.00 : ffff800010121a1c: b.cc ffff800010121c3c // b.lo, b.ul, b.last : if (delta_exec < sysctl_sched_min_granularity) 0.00 : ffff800010121a20: adrp x2, ffff800011920000 0.00 : ffff800010121a24: ldr w2, [x2, #2268] 0.00 : ffff800010121a28: cmp x1, x2 0.00 : ffff800010121a2c: b.cc ffff800010121a58 // b.lo, b.ul, b.last : __pick_first_entity(): : struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 0.00 : ffff800010121a30: ldr x1, [x19, #64] : check_preempt_tick(): : delta = curr->vruntime - se->vruntime; 0.00 : ffff800010121a34: ldr x2, [x20, #88] : __pick_first_entity(): : return rb_entry(left, struct sched_entity, run_node); 0.00 : ffff800010121a38: cmp x1, #0x0 0.00 : ffff800010121a3c: sub x1, x1, #0x18 0.00 : ffff800010121a40: csel x1, x1, xzr, ne // ne = any : check_preempt_tick(): : delta = curr->vruntime - se->vruntime; 0.00 : ffff800010121a44: ldr x1, [x1, #88] : if (delta > ideal_runtime) 0.00 : ffff800010121a48: subs x2, x2, x1 0.00 : ffff800010121a4c: ccmp x0, x2, #0x2, pl // pl = nfrst 0.00 : ffff800010121a50: b.cc ffff800010121c6c // b.lo, b.ul, b.last 0.00 : ffff800010121a54: nop : task_tick_fair(): : for_each_sched_entity(se) { 0.00 : ffff800010121a58: ldr x20, [x20, #120] 0.00 : ffff800010121a5c: cbnz x20, ffff800010121708 0.00 : ffff800010121a60: ldr x19, [x29, #16] 0.00 : ffff800010121a64: ldp x21, x22, [x29, #32] 0.00 : ffff800010121a68: ldp x25, x26, [x29, #64] 0.00 : ffff800010121a6c: ldp x27, x28, [x29, #80] : arch_static_branch(): : asm_volatile_goto( 0.00 : ffff800010121a70: nop 0.00 : ffff800010121a74: nop 0.00 : ffff800010121a78: adrp x6, ffff800011528000 : task_tick_fair(): : : if (static_branch_unlikely(&sched_numa_balancing)) : task_tick_numa(rq, curr); : : update_misfit_status(curr, rq); : update_overutilized_status(task_rq(curr)); 0.00 : ffff800010121a7c: ldr x0, [x29, #112] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121a80: ldr w1, [x23, #68] : task_tick_fair(): 0.00 : ffff800010121a84: add x24, x0, #0x928 0.00 : ffff800010121a88: add x0, x6, #0xe80 0.00 : ffff800010121a8c: ldr x1, [x24, w1, uxtw #3] 0.00 : ffff800010121a90: add x1, x0, x1 0.00 : ffff800010121a94: ldr x3, [x1, #2464] : __read_once_size(): 0.00 : ffff800010121a98: ldr w2, [x3, #92] : update_overutilized_status(): : if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) { 0.00 : ffff800010121a9c: cbnz w2, ffff800010121ae4 : cpu_util(): : cfs_rq = &cpu_rq(cpu)->cfs; 0.00 : ffff800010121aa0: ldrsw x1, [x1, #2568] 0.00 : ffff800010121aa4: ldr x1, [x24, x1, lsl #3] 0.00 : ffff800010121aa8: add x0, x0, x1 : __read_once_size(): 0.00 : ffff800010121aac: ldr x2, [x0, #304] 0.00 : ffff800010121ab0: ldr w5, [x0, #312] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff800010121ab4: ldr x1, [x0, #2488] : util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); 0.00 : ffff800010121ab8: cmp w5, w2 : cpu_overutilized(): : return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); 0.00 : ffff800010121abc: ldr x4, [x0, #2480] : cpu_util(): : return min_t(unsigned long, util, capacity_orig_of(cpu)); 0.00 : ffff800010121ac0: csel w0, w2, w5, ls // ls = plast 0.00 : ffff800010121ac4: cmp x1, x0 0.00 : ffff800010121ac8: csel x0, x1, x0, ls // ls = plast : cpu_overutilized(): : return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); 0.00 : ffff800010121acc: lsl x2, x4, #10 0.00 : ffff800010121ad0: add x0, x0, x0, lsl #2 : update_overutilized_status(): : if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) { 0.00 : ffff800010121ad4: cmp x2, x0, lsl #8 0.00 : ffff800010121ad8: b.hi ffff800010121ae4 // b.pmore : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010121adc: mov w0, #0x2 // #2 0.00 : ffff800010121ae0: str w0, [x3, #92] : task_tick_fair(): : } 0.00 : ffff800010121ae4: ldr x20, [sp, #24] 100.00 : ffff800010121ae8: ldp x23, x24, [sp, #48] 0.00 : ffff800010121aec: ldp x29, x30, [sp], #128 0.00 : ffff800010121af0: ret : update_load_avg(): : } else if (decayed) { 0.00 : ffff800010121af4: cbnz w0, ffff800010121988 : entity_tick(): : update_cfs_group(curr); 0.00 : ffff800010121af8: mov x0, x20 0.00 : ffff800010121afc: bl ffff80001011fd78 : resched_curr(rq_of(cfs_rq)); 0.00 : ffff800010121b00: ldr x0, [x19, #304] : if (queued) { 0.00 : ffff800010121b04: cbz w22, ffff8000101219e8 : check_preempt_tick(): : resched_curr(rq_of(cfs_rq)); 0.00 : ffff800010121b08: bl ffff800010114bb8 : task_tick_fair(): : for_each_sched_entity(se) { 0.00 : ffff800010121b0c: ldr x20, [x20, #120] 0.00 : ffff800010121b10: cbnz x20, ffff800010121708 0.00 : ffff800010121b14: b ffff800010121a60 : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff800010121b18: ldr x0, [x29, #112] 0.00 : ffff800010121b1c: ldrsw x3, [x1, #2568] 0.00 : ffff800010121b20: add x2, x0, #0x928 0.00 : ffff800010121b24: ldr x0, [x29, #104] 0.00 : ffff800010121b28: ldr x2, [x2, x3, lsl #3] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010121b2c: ldr x0, [x0, x2] : cpufreq_update_util(): : cpu_of(rq))); : if (data) 0.00 : ffff800010121b30: cbz x0, ffff800010121998 : data->func(data, rq_clock(rq), flags); 0.00 : ffff800010121b34: ldr x3, [x0] 0.00 : ffff800010121b38: mov w2, #0x0 // #0 0.00 : ffff800010121b3c: ldr x1, [x1, #2400] 0.00 : ffff800010121b40: blr x3 0.00 : ffff800010121b44: b ffff800010121998 : task_tick_numa(): : if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) 0.00 : ffff800010121b48: ldr x2, [x23, #952] 0.00 : ffff800010121b4c: cbz x2, ffff800010121a74 0.00 : ffff800010121b50: ldr w0, [x23, #44] 0.00 : ffff800010121b54: tbnz w0, #2, ffff800010121a74 0.00 : ffff800010121b58: str x19, [x29, #16] : struct callback_head *work = &curr->numa_work; 0.00 : ffff800010121b5c: add x19, x23, #0x8b8 : if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) 0.00 : ffff800010121b60: ldr x0, [x23, #2232] 0.00 : ffff800010121b64: cmp x19, x0 0.00 : ffff800010121b68: b.eq ffff800010121c84 // b.none 0.00 : ffff800010121b6c: ldr x19, [x29, #16] 0.00 : ffff800010121b70: b ffff800010121a74 : update_misfit_status(): : if (!p) { 0.00 : ffff800010121b74: cbz x23, ffff800010121c5c : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010121b78: ldr x0, [x29, #96] 0.00 : ffff800010121b7c: adrp x6, ffff800011528000 : __read_once_size(): 0.00 : ffff800010121b80: ldr x4, [x23, #432] : capacity_of(): 0.00 : ffff800010121b84: add x1, x6, #0xe80 : __read_once_size(): 0.00 : ffff800010121b88: ldr x3, [x23, #440] : capacity_of(): 0.00 : ffff800010121b8c: ldrsw x5, [x0, #2568] 0.00 : ffff800010121b90: ldr x0, [x29, #112] 0.00 : ffff800010121b94: add x2, x0, #0x928 : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff800010121b98: lsr x0, x3, #32 0.00 : ffff800010121b9c: cmp w0, w3 0.00 : ffff800010121ba0: csel w0, w0, w3, hi // hi = pmore : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010121ba4: ldr x2, [x2, x5, lsl #3] : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff800010121ba8: orr w0, w0, #0x1 : task_util_est(): : return max(task_util(p), _task_util_est(p)); 0.00 : ffff800010121bac: cmp x0, x4 : capacity_of(): : return cpu_rq(cpu)->cpu_capacity; 0.00 : ffff800010121bb0: add x1, x2, x1 : task_util_est(): : return max(task_util(p), _task_util_est(p)); 0.00 : ffff800010121bb4: csel x0, x0, x4, cs // cs = hs, nlast : task_fits_capacity(): : return fits_capacity(uclamp_task_util(p), capacity); 0.00 : ffff800010121bb8: add x0, x0, x0, lsl #2 0.00 : ffff800010121bbc: ldr x1, [x1, #2480] 0.00 : ffff800010121bc0: lsl x1, x1, #10 : update_misfit_status(): : if (task_fits_capacity(p, capacity_of(cpu_of(rq)))) { 0.00 : ffff800010121bc4: cmp x1, x0, lsl #8 0.00 : ffff800010121bc8: b.hi ffff800010121c60 // b.pmore : task_cfs_rq(): : return p->se.cfs_rq; 0.00 : ffff800010121bcc: ldr x7, [x23, #320] : task_h_load(): : update_cfs_rq_h_load(cfs_rq); 0.00 : ffff800010121bd0: mov x0, x7 0.00 : ffff800010121bd4: bl ffff80001011c898 : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 0.00 : ffff800010121bd8: ldr x2, [x7, #280] 0.00 : ffff800010121bdc: ldr x0, [x23, #416] 0.00 : ffff800010121be0: ldr x1, [x7, #160] 0.00 : ffff800010121be4: add x1, x1, #0x1 0.00 : ffff800010121be8: mul x0, x0, x2 : div64_u64(): : * : * Return: dividend / divisor : */ : static inline u64 div64_u64(u64 dividend, u64 divisor) : { : return dividend / divisor; 0.00 : ffff800010121bec: udiv x0, x0, x1 : update_misfit_status(): : rq->misfit_task_load = task_h_load(p); 0.00 : ffff800010121bf0: ldr x1, [x29, #96] 0.00 : ffff800010121bf4: str x0, [x1, #2512] 0.00 : ffff800010121bf8: b ffff800010121a7c : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff800010121bfc: add x0, x0, #0x140 0.00 : ffff800010121c00: b ffff800010128a7c : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff800010121c04: ldr x0, [x19, #160] 0.00 : ffff800010121c08: str x0, [x19, #256] 0.00 : ffff800010121c0c: b ffff8000101219d8 : update_tg_cfs_runnable(): : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010121c10: ldr x0, [x1] : u64 runnable_load_sum, load_sum = 0; 0.00 : ffff800010121c14: mov x3, #0x0 // #0 : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010121c18: lsr x0, x0, #10 0.00 : ffff800010121c1c: cbz x0, ffff800010121c2c : div_s64_rem(): : return dividend / divisor; 0.00 : ffff800010121c20: ldr x3, [x1, #136] : *remainder = dividend % divisor; 0.00 : ffff800010121c24: sxtw x0, w0 : return dividend / divisor; 0.00 : ffff800010121c28: sdiv x3, x3, x0 : update_tg_cfs_runnable(): : runnable_sum = min(se->avg.load_sum, load_sum); 0.00 : ffff800010121c2c: ldr x0, [x20, #200] 0.00 : ffff800010121c30: cmp x0, x3 0.00 : ffff800010121c34: csel x3, x0, x3, ls // ls = plast 0.00 : ffff800010121c38: b ffff80001012188c : check_preempt_tick(): : resched_curr(rq_of(cfs_rq)); 0.00 : ffff800010121c3c: ldr x0, [x19, #304] 0.00 : ffff800010121c40: bl ffff800010114bb8 : clear_buddies(cfs_rq, curr); 0.00 : ffff800010121c44: mov x1, x20 0.00 : ffff800010121c48: mov x0, x19 0.00 : ffff800010121c4c: bl ffff80001011cfb0 : task_tick_fair(): : for_each_sched_entity(se) { 0.00 : ffff800010121c50: ldr x20, [x20, #120] 0.00 : ffff800010121c54: cbnz x20, ffff800010121708 0.00 : ffff800010121c58: b ffff800010121a60 0.00 : ffff800010121c5c: adrp x6, ffff800011528000 : update_misfit_status(): : rq->misfit_task_load = 0; 0.00 : ffff800010121c60: ldr x0, [x29, #96] 0.00 : ffff800010121c64: str xzr, [x0, #2512] 0.00 : ffff800010121c68: b ffff800010121a7c : check_preempt_tick(): : resched_curr(rq_of(cfs_rq)); 0.00 : ffff800010121c6c: ldr x0, [x19, #304] 0.00 : ffff800010121c70: bl ffff800010114bb8 0.00 : ffff800010121c74: b ffff800010121b0c 0.00 : ffff800010121c78: adrp x0, ffff800011909000 0.00 : ffff800010121c7c: str x0, [x29, #112] 0.00 : ffff800010121c80: b ffff800010121a70 : task_tick_numa(): : if (now > curr->node_stamp + period) { 0.00 : ffff800010121c84: ldr w20, [x23, #2184] 0.00 : ffff800010121c88: mov w3, #0x4240 // #16960 0.00 : ffff800010121c8c: ldr x0, [x23, #2208] 0.00 : ffff800010121c90: movk w3, #0xf, lsl #16 0.00 : ffff800010121c94: ldr x1, [x23, #272] 0.00 : ffff800010121c98: umaddl x20, w20, w3, x0 0.00 : ffff800010121c9c: cmp x1, x20 0.00 : ffff800010121ca0: b.ls ffff800010121b6c // b.plast : if (!curr->node_stamp) 0.00 : ffff800010121ca4: cbnz x0, ffff800010121cc0 : curr->numa_scan_period = task_scan_start(curr); 0.00 : ffff800010121ca8: mov x0, x23 0.00 : ffff800010121cac: bl ffff80001011df38 0.00 : ffff800010121cb0: ldr x1, [x23, #2208] 0.00 : ffff800010121cb4: ldr x2, [x23, #952] 0.00 : ffff800010121cb8: add x20, x20, x1 0.00 : ffff800010121cbc: str w0, [x23, #2184] : if (!time_before(jiffies, curr->mm->numa_next_scan)) 0.00 : ffff800010121cc0: adrp x0, ffff800011907000 : curr->node_stamp += period; 0.00 : ffff800010121cc4: str x20, [x23, #2208] : if (!time_before(jiffies, curr->mm->numa_next_scan)) 0.00 : ffff800010121cc8: ldr x0, [x0, #2432] 0.00 : ffff800010121ccc: ldr x1, [x2, #824] 0.00 : ffff800010121cd0: sub x0, x0, x1 0.00 : ffff800010121cd4: tbnz x0, #63, ffff800010121b6c : task_work_add(curr, work, true); 0.00 : ffff800010121cd8: mov x1, x19 0.00 : ffff800010121cdc: mov w2, #0x1 // #1 0.00 : ffff800010121ce0: mov x0, x23 0.00 : ffff800010121ce4: bl ffff80001010a180 0.00 : ffff800010121ce8: ldr x19, [x29, #16] 0.00 : ffff800010121cec: b ffff800010121a74 Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cd2e28 : : preempt_schedule(): : #ifdef CONFIG_PREEMPTION : /* : * This is the entry point to schedule() from in-kernel preemption : * off of preempt_enable. : */ : asmlinkage __visible void __sched notrace preempt_schedule(void) 51.76 : ffff800010cd2e28: stp x29, x30, [sp, #-16]! 0.00 : ffff800010cd2e2c: mov x29, sp : arch_local_save_flags(): : */ : static inline unsigned long arch_local_save_flags(void) : { : unsigned long flags; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010cd2e30: mrs x1, daif : arch_irqs_disabled_flags(): : : static inline int arch_irqs_disabled_flags(unsigned long flags) : { : int res; : : asm volatile(ALTERNATIVE( 0.00 : ffff800010cd2e34: and w0, w1, #0x80 : preempt_schedule(): : { : /* : * If there is a non-zero preempt_count or interrupts are disabled, : * we do not want to preempt the current task. Just return.. : */ : if (likely(!preemptible())) 0.00 : ffff800010cd2e38: cbz w0, ffff800010cd2e44 : return; : : preempt_schedule_common(); : } 0.00 : ffff800010cd2e3c: ldp x29, x30, [sp], #16 0.00 : ffff800010cd2e40: ret : preempt_schedule_common(); 48.24 : ffff800010cd2e44: bl ffff800010cd2de0 : } 0.00 : ffff800010cd2e48: ldp x29, x30, [sp], #16 0.00 : ffff800010cd2e4c: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010093ae8 : : cpu_have_feature(): : } : EXPORT_SYMBOL_GPL(cpu_set_feature); : : bool cpu_have_feature(unsigned int num) : { : WARN_ON(num >= MAX_CPU_FEATURES); 100.00 : ffff800010093ae8: cmp w0, #0x3f 0.00 : ffff800010093aec: b.hi ffff800010093b0c // b.pmore : return elf_hwcap & BIT(num); 0.00 : ffff800010093af0: adrp x2, ffff800011909000 0.00 : ffff800010093af4: mov x1, #0x1 // #1 0.00 : ffff800010093af8: lsl x0, x1, x0 0.00 : ffff800010093afc: ldr x1, [x2, #2328] 0.00 : ffff800010093b00: tst x0, x1 : } 0.00 : ffff800010093b04: cset w0, ne // ne = any 0.00 : ffff800010093b08: ret : WARN_ON(num >= MAX_CPU_FEATURES); 0.00 : ffff800010093b0c: brk #0x800 : return elf_hwcap & BIT(num); 0.00 : ffff800010093b10: adrp x2, ffff800011909000 0.00 : ffff800010093b14: mov x1, #0x1 // #1 0.00 : ffff800010093b18: lsl x0, x1, x0 0.00 : ffff800010093b1c: ldr x1, [x2, #2328] 0.00 : ffff800010093b20: tst x0, x1 : } 0.00 : ffff800010093b24: cset w0, ne // ne = any 0.00 : ffff800010093b28: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011e648 : : task_numa_work(): : /* : * The expensive part of numa migration is done from task_work context. : * Triggered from task_tick_numa(). : */ : static void task_numa_work(struct callback_head *work) : { 0.00 : ffff80001011e648: stp x29, x30, [sp, #-112]! : unsigned long migrate, next_scan, now = jiffies; 0.00 : ffff80001011e64c: adrp x1, ffff800011907000 : { 0.00 : ffff80001011e650: mov x29, sp 0.00 : ffff80001011e654: stp x19, x20, [sp, #16] 0.00 : ffff80001011e658: str x21, [sp, #32] 0.00 : ffff80001011e65c: str x23, [sp, #48] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001011e660: mrs x19, sp_el0 : task_numa_work(): : unsigned long migrate, next_scan, now = jiffies; 0.00 : ffff80001011e664: ldr x21, [x1, #2432] : struct task_struct *p = current; : struct mm_struct *mm = p->mm; : u64 runtime = p->se.sum_exec_runtime; 0.00 : ffff80001011e668: ldr x23, [x19, #272] : struct mm_struct *mm = p->mm; 0.00 : ffff80001011e66c: ldr x20, [x19, #952] : unsigned long nr_pte_updates = 0; : long pages, virtpages; : : SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); : : work->next = work; 0.00 : ffff80001011e670: str x0, [x0] : * NOTE: make sure not to dereference p->mm before this check, : * exit_task_work() happens _after_ exit_mm() so we could be called : * without p->mm even though we still had it when we enqueued this : * work. : */ : if (p->flags & PF_EXITING) 0.00 : ffff80001011e674: ldr w0, [x19, #44] 0.00 : ffff80001011e678: tbnz w0, #2, ffff80001011e6e0 0.00 : ffff80001011e67c: str x22, [x29, #40] : return; : : if (!mm->numa_next_scan) { 0.00 : ffff80001011e680: ldr x22, [x20, #824] 0.00 : ffff80001011e684: cbnz x22, ffff80001011e69c : mm->numa_next_scan = now + : msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 0.00 : ffff80001011e688: adrp x0, ffff800011920000 : msecs_to_jiffies(): : if (__builtin_constant_p(m)) { : if ((int)m < 0) : return MAX_JIFFY_OFFSET; : return _msecs_to_jiffies(m); : } else { : return __msecs_to_jiffies(m); 0.00 : ffff80001011e68c: ldr w0, [x0, #2284] 0.00 : ffff80001011e690: bl ffff800010167188 <__msecs_to_jiffies> : task_numa_work(): : mm->numa_next_scan = now + 0.00 : ffff80001011e694: add x22, x21, x0 0.00 : ffff80001011e698: str x22, [x20, #824] : : /* : * Enforce maximal scan/migration frequency.. : */ : migrate = mm->numa_next_scan; : if (time_before(now, migrate)) 0.00 : ffff80001011e69c: sub x0, x21, x22 0.00 : ffff80001011e6a0: tbnz x0, #63, ffff80001011e6dc : return; : : if (p->numa_scan_period == 0) { 0.00 : ffff80001011e6a4: ldr w0, [x19, #2184] 0.00 : ffff80001011e6a8: cbz w0, ffff80001011e734 : msecs_to_jiffies(): 0.00 : ffff80001011e6ac: bl ffff800010167188 <__msecs_to_jiffies> : task_numa_work(): : p->numa_scan_period_max = task_scan_max(p); : p->numa_scan_period = task_scan_start(p); : } : : next_scan = now + msecs_to_jiffies(p->numa_scan_period); 0.00 : ffff80001011e6b0: add x2, x21, x0 : if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) 0.00 : ffff80001011e6b4: add x1, x20, #0x338 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011e6b8: b ffff80001011e6f4 0.00 : ffff80001011e6bc: b ffff80001011e6f4 : __lse__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, l, "memory") : __CMPXCHG_CASE(x, , rel_, 64, l, "memory") : __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") : __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") : __CMPXCHG_CASE(w, , mb_, 32, al, "memory") : __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 0.00 : ffff80001011e6c0: mov x0, x1 0.00 : ffff80001011e6c4: mov x1, x22 0.00 : ffff80001011e6c8: mov x4, x1 0.00 : ffff80001011e6cc: casal x4, x2, [x0] 0.00 : ffff80001011e6d0: mov x0, x4 : task_numa_work(): 0.00 : ffff80001011e6d4: cmp x0, x22 0.00 : ffff80001011e6d8: b.eq ffff80001011e704 // b.none 0.00 : ffff80001011e6dc: ldr x22, [x29, #40] : */ : if (unlikely(p->se.sum_exec_runtime != runtime)) { : u64 diff = p->se.sum_exec_runtime - runtime; : p->node_stamp += 32 * diff; : } : } 0.00 : ffff80001011e6e0: ldp x19, x20, [sp, #16] 0.00 : ffff80001011e6e4: ldr x21, [sp, #32] 0.00 : ffff80001011e6e8: ldr x23, [sp, #48] 0.00 : ffff80001011e6ec: ldp x29, x30, [sp], #112 0.00 : ffff80001011e6f0: ret : __ll_sc__cmpxchg_case_mb_64(): : __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) : __CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) : __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) : __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) : __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 0.00 : ffff80001011e6f4: add x3, x20, #0x338 0.00 : ffff80001011e6f8: b ffff800010128954 : task_numa_work(): : if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) 0.00 : ffff80001011e6fc: cmp x0, x22 0.00 : ffff80001011e700: b.ne ffff80001011e6dc // b.any 0.00 : ffff80001011e704: str x27, [x29, #80] : pages = sysctl_numa_balancing_scan_size; 0.00 : ffff80001011e708: adrp x1, ffff800011920000 : p->node_stamp += 2 * TICK_NSEC; 0.00 : ffff80001011e70c: ldr x0, [x19, #2208] : pages = sysctl_numa_balancing_scan_size; 0.00 : ffff80001011e710: ldr w6, [x1, #2256] : p->node_stamp += 2 * TICK_NSEC; 0.00 : ffff80001011e714: add x0, x0, #0x7a1, lsl #12 0.00 : ffff80001011e718: add x0, x0, #0x200 0.00 : ffff80001011e71c: str x0, [x19, #2208] : pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 0.00 : ffff80001011e720: lsl x27, x6, #8 : if (!pages) 0.00 : ffff80001011e724: cbnz x27, ffff80001011e750 0.00 : ffff80001011e728: ldr x22, [x29, #40] 0.00 : ffff80001011e72c: ldr x27, [x29, #80] 0.00 : ffff80001011e730: b ffff80001011e6e0 : p->numa_scan_period_max = task_scan_max(p); 0.00 : ffff80001011e734: mov x0, x19 0.00 : ffff80001011e738: bl ffff80001011cc58 0.00 : ffff80001011e73c: str w0, [x19, #2188] : p->numa_scan_period = task_scan_start(p); 0.00 : ffff80001011e740: mov x0, x19 0.00 : ffff80001011e744: bl ffff80001011df38 0.00 : ffff80001011e748: str w0, [x19, #2184] 0.00 : ffff80001011e74c: b ffff80001011e6ac 0.00 : ffff80001011e750: str x28, [x29, #88] : if (!down_read_trylock(&mm->mmap_sem)) 0.00 : ffff80001011e754: add x0, x20, #0x68 0.00 : ffff80001011e758: str x0, [x29, #104] : start = mm->numa_scan_offset; 0.00 : ffff80001011e75c: ldr x28, [x20, #832] : if (!down_read_trylock(&mm->mmap_sem)) 0.00 : ffff80001011e760: bl ffff800010138678 0.00 : ffff80001011e764: cbnz w0, ffff80001011e774 0.00 : ffff80001011e768: ldr x22, [x29, #40] 0.00 : ffff80001011e76c: ldp x27, x28, [x29, #80] 0.00 : ffff80001011e770: b ffff80001011e6e0 0.00 : ffff80001011e774: str x26, [x29, #72] : vma = find_vma(mm, start); 0.00 : ffff80001011e778: mov x1, x28 0.00 : ffff80001011e77c: mov x0, x20 0.00 : ffff80001011e780: bl ffff800010212848 0.00 : ffff80001011e784: mov x26, x0 : if (!vma) { 0.00 : ffff80001011e788: cbz x0, ffff80001011e8e8 0.00 : ffff80001011e78c: str x24, [x29, #56] : vma_migratable(): : * do so then migration (at least from node to node) is not : * possible. : */ : if (vma->vm_file && : gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping)) : < policy_zone) 0.00 : ffff80001011e790: adrp x24, ffff800011b2c000 : task_numa_work(): : virtpages = pages * 8; /* Scan up to this much virtual space */ 0.00 : ffff80001011e794: lsl x21, x27, #3 : vma_migratable(): 0.00 : ffff80001011e798: add x24, x24, #0x5a8 : if (vma->vm_flags & (VM_IO | VM_PFNMAP)) 0.00 : ffff80001011e79c: mov x22, #0x4400 // #17408 0.00 : ffff80001011e7a0: b ffff80001011e7ac : task_numa_work(): : for (; vma; vma = vma->vm_next) { 0.00 : ffff80001011e7a4: ldr x26, [x26, #16] 0.00 : ffff80001011e7a8: cbz x26, ffff80001011e8c8 : vma_migratable(): 0.00 : ffff80001011e7ac: ldr x0, [x26, #80] 0.00 : ffff80001011e7b0: tst x0, x22 0.00 : ffff80001011e7b4: b.ne ffff80001011e7a4 // b.any : if (vma->vm_file && 0.00 : ffff80001011e7b8: ldr x0, [x26, #160] 0.00 : ffff80001011e7bc: cbz x0, ffff80001011e7e8 : gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping)) 0.00 : ffff80001011e7c0: ldr x0, [x0, #240] : gfp_zone(): : static inline enum zone_type gfp_zone(gfp_t flags) : { : enum zone_type z; : int bit = (__force int) (flags & GFP_ZONEMASK); : : z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) & 0.00 : ffff80001011e7c4: mov w2, #0x122 // #290 0.00 : ffff80001011e7c8: movk w2, #0x132, lsl #16 : vma_migratable(): : if (vma->vm_file && 0.00 : ffff80001011e7cc: ldr w1, [x24] : gfp_zone(): : int bit = (__force int) (flags & GFP_ZONEMASK); 0.00 : ffff80001011e7d0: ldr w0, [x0, #24] : z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) & 0.00 : ffff80001011e7d4: ubfiz w0, w0, #1, #4 0.00 : ffff80001011e7d8: asr w0, w2, w0 0.00 : ffff80001011e7dc: and w0, w0, #0x3 : vma_migratable(): 100.00 : ffff80001011e7e0: cmp w0, w1 0.00 : ffff80001011e7e4: b.cc ffff80001011e7a4 // b.lo, b.ul, b.last : task_numa_work(): : if (!vma_migratable(vma) || !vma_policy_mof(vma) || 0.00 : ffff80001011e7e8: mov x0, x26 0.00 : ffff80001011e7ec: bl ffff800010244b88 0.00 : ffff80001011e7f0: tst w0, #0xff 0.00 : ffff80001011e7f4: b.eq ffff80001011e7a4 // b.none : is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 0.00 : ffff80001011e7f8: ldr x0, [x26, #80] 0.00 : ffff80001011e7fc: mov x1, #0x10400000 // #272629760 0.00 : ffff80001011e800: tst x0, x1 0.00 : ffff80001011e804: b.ne ffff80001011e7a4 // b.any : if (!vma->vm_mm || 0.00 : ffff80001011e808: ldr x1, [x26, #64] 0.00 : ffff80001011e80c: cbz x1, ffff80001011e7a4 0.00 : ffff80001011e810: ldr x1, [x26, #160] 0.00 : ffff80001011e814: cbz x1, ffff80001011e824 : (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) 0.00 : ffff80001011e818: and x1, x0, #0x3 0.00 : ffff80001011e81c: cmp x1, #0x1 0.00 : ffff80001011e820: b.eq ffff80001011e7a4 // b.none : if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) 0.00 : ffff80001011e824: tst x0, #0x7 0.00 : ffff80001011e828: b.eq ffff80001011e7a4 // b.none 0.00 : ffff80001011e82c: str x25, [x29, #64] 0.00 : ffff80001011e830: ldr x2, [x26, #8] 0.00 : ffff80001011e834: b ffff80001011e844 : } while (end != vma->vm_end); 0.00 : ffff80001011e838: ldr x2, [x26, #8] 0.00 : ffff80001011e83c: cmp x2, x28 0.00 : ffff80001011e840: b.eq ffff80001011e914 // b.none : start = max(start, vma->vm_start); 0.00 : ffff80001011e844: ldr x5, [x26] : end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 0.00 : ffff80001011e848: mov x1, #0x1fffff // #2097151 : nr_pte_updates = change_prot_numa(vma, start, end); 0.00 : ffff80001011e84c: mov x0, x26 : start = max(start, vma->vm_start); 0.00 : ffff80001011e850: cmp x5, x28 0.00 : ffff80001011e854: csel x25, x5, x28, cs // cs = hs, nlast : end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 0.00 : ffff80001011e858: add x4, x25, x1 : nr_pte_updates = change_prot_numa(vma, start, end); 0.00 : ffff80001011e85c: mov x1, x25 : end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 0.00 : ffff80001011e860: add x4, x4, x27, lsl #12 0.00 : ffff80001011e864: and x4, x4, #0xffffffffffe00000 : end = min(end, vma->vm_end); 0.00 : ffff80001011e868: cmp x4, x2 0.00 : ffff80001011e86c: csel x28, x4, x2, ls // ls = plast : nr_pte_updates = change_prot_numa(vma, start, end); 0.00 : ffff80001011e870: mov x2, x28 0.00 : ffff80001011e874: bl ffff800010243f58 : pages -= (end - start) >> PAGE_SHIFT; 0.00 : ffff80001011e878: cmp x0, #0x0 0.00 : ffff80001011e87c: sub x5, x28, x25 0.00 : ffff80001011e880: lsr x5, x5, #12 0.00 : ffff80001011e884: sub x0, x27, x5 : virtpages -= (end - start) >> PAGE_SHIFT; 0.00 : ffff80001011e888: sub x21, x21, x5 : pages -= (end - start) >> PAGE_SHIFT; 0.00 : ffff80001011e88c: csel x27, x0, x27, ne // ne = any : if (pages <= 0 || virtpages <= 0) 0.00 : ffff80001011e890: cmp x27, #0x0 0.00 : ffff80001011e894: ccmp x21, #0x0, #0x4, gt 0.00 : ffff80001011e898: b.gt ffff80001011e838 : mm->numa_scan_offset = start; 0.00 : ffff80001011e89c: str x28, [x20, #832] 0.00 : ffff80001011e8a0: ldp x24, x25, [x29, #56] : up_read(&mm->mmap_sem); 0.00 : ffff80001011e8a4: ldr x0, [x29, #104] 0.00 : ffff80001011e8a8: bl ffff800010138db0 : if (unlikely(p->se.sum_exec_runtime != runtime)) { 0.00 : ffff80001011e8ac: ldr x0, [x19, #272] 0.00 : ffff80001011e8b0: cmp x0, x23 0.00 : ffff80001011e8b4: b.ne ffff80001011e91c // b.any 0.00 : ffff80001011e8b8: ldr x22, [x29, #40] 0.00 : ffff80001011e8bc: ldp x26, x27, [x29, #72] 0.00 : ffff80001011e8c0: ldr x28, [x29, #88] 0.00 : ffff80001011e8c4: b ffff80001011e6e0 : out: 0.00 : ffff80001011e8c8: ldr x24, [x29, #56] : reset_ptenuma_scan(): : WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 0.00 : ffff80001011e8cc: ldr x1, [x19, #952] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001011e8d0: ldr w0, [x1, #840] : reset_ptenuma_scan(): 0.00 : ffff80001011e8d4: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001011e8d8: str w0, [x1, #840] : reset_ptenuma_scan(): : p->mm->numa_scan_offset = 0; 0.00 : ffff80001011e8dc: ldr x0, [x19, #952] 0.00 : ffff80001011e8e0: str xzr, [x0, #832] 0.00 : ffff80001011e8e4: b ffff80001011e8a4 : WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 0.00 : ffff80001011e8e8: ldr x1, [x19, #952] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001011e8ec: ldr w0, [x1, #840] : reset_ptenuma_scan(): 0.00 : ffff80001011e8f0: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff80001011e8f4: str w0, [x1, #840] : reset_ptenuma_scan(): : p->mm->numa_scan_offset = 0; 0.00 : ffff80001011e8f8: ldr x0, [x19, #952] 0.00 : ffff80001011e8fc: str xzr, [x0, #832] : task_numa_work(): : vma = mm->mmap; 0.00 : ffff80001011e900: ldr x26, [x20] : for (; vma; vma = vma->vm_next) { 0.00 : ffff80001011e904: cbz x26, ffff80001011e8cc : start = 0; 0.00 : ffff80001011e908: mov x28, #0x0 // #0 0.00 : ffff80001011e90c: str x24, [x29, #56] 0.00 : ffff80001011e910: b ffff80001011e790 0.00 : ffff80001011e914: ldr x25, [x29, #64] 0.00 : ffff80001011e918: b ffff80001011e7a4 : p->node_stamp += 32 * diff; 0.00 : ffff80001011e91c: ldr x1, [x19, #2208] : u64 diff = p->se.sum_exec_runtime - runtime; 0.00 : ffff80001011e920: sub x0, x0, x23 : p->node_stamp += 32 * diff; 0.00 : ffff80001011e924: add x0, x1, x0, lsl #5 0.00 : ffff80001011e928: str x0, [x19, #2208] 0.00 : ffff80001011e92c: ldr x22, [x29, #40] 0.00 : ffff80001011e930: ldp x26, x27, [x29, #72] 0.00 : ffff80001011e934: ldr x28, [x29, #88] 0.00 : ffff80001011e938: b ffff80001011e6e0 Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010218048 : : change_protection(): : } : : unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, : unsigned long end, pgprot_t newprot, : int dirty_accountable, int prot_numa) : { 0.00 : ffff800010218048: stp x29, x30, [sp, #-304]! 0.00 : ffff80001021804c: adrp x6, ffff800011909000 0.00 : ffff800010218050: add x6, x6, #0x908 0.00 : ffff800010218054: mov x29, sp 0.00 : ffff800010218058: ldr x7, [x6] 0.00 : ffff80001021805c: str x7, [x29, #296] 0.00 : ffff800010218060: mov x7, #0x0 // #0 : is_vm_hugetlb_page(): : : #include : : static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma) : { : return !!(vma->vm_flags & VM_HUGETLB); 0.00 : ffff800010218064: ldr x6, [x0, #80] : change_protection(): 0.00 : ffff800010218068: str x1, [x29, #128] 0.00 : ffff80001021806c: str x2, [x29, #144] 0.00 : ffff800010218070: str x3, [x29, #168] 0.00 : ffff800010218074: str w4, [x29, #184] : unsigned long pages; : : if (is_vm_hugetlb_page(vma)) 0.00 : ffff800010218078: tbnz w6, #22, ffff800010218774 0.00 : ffff80001021807c: str x28, [x29, #88] 0.00 : ffff800010218080: mov x28, x0 : change_protection_range(): : BUG_ON(addr >= end); 0.00 : ffff800010218084: ldr x1, [x29, #144] : struct mm_struct *mm = vma->vm_mm; 0.00 : ffff800010218088: ldr x0, [x0, #64] 0.00 : ffff80001021808c: stp x19, x20, [x29, #16] 0.00 : ffff800010218090: stp x21, x22, [x29, #32] 0.00 : ffff800010218094: stp x23, x24, [x29, #48] 0.00 : ffff800010218098: stp x25, x26, [x29, #64] 0.00 : ffff80001021809c: str x27, [x29, #80] 0.00 : ffff8000102180a0: str x0, [x29, #120] : BUG_ON(addr >= end); 0.00 : ffff8000102180a4: ldr x0, [x29, #128] 0.00 : ffff8000102180a8: cmp x0, x1 0.00 : ffff8000102180ac: b.cs ffff80001021883c // b.hs, b.nlast : pgd = pgd_offset(mm, addr); 0.00 : ffff8000102180b0: ldp x2, x0, [x29, #120] 0.00 : ffff8000102180b4: mov w27, w5 : inc_tlb_flush_pending(): : atomic_set(&mm->tlb_flush_pending, 0); : } : : static inline void inc_tlb_flush_pending(struct mm_struct *mm) : { : atomic_inc(&mm->tlb_flush_pending); 0.00 : ffff8000102180b8: add x3, x2, #0x34c 0.00 : ffff8000102180bc: str x3, [x29, #112] : change_protection_range(): 0.00 : ffff8000102180c0: ldr x1, [x2, #64] 0.00 : ffff8000102180c4: ubfx x0, x0, #39, #9 0.00 : ffff8000102180c8: add x0, x1, x0, lsl #3 0.00 : ffff8000102180cc: str x0, [x29, #160] : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000102180d0: b ffff800010218780 0.00 : ffff8000102180d4: b ffff800010218780 : __lse_atomic_add(): : } : : ATOMIC_OP(andnot, stclr) : ATOMIC_OP(or, stset) : ATOMIC_OP(xor, steor) : ATOMIC_OP(add, stadd) 0.00 : ffff8000102180d8: mov w0, #0x1 // #1 0.00 : ffff8000102180dc: stadd w0, [x3] 0.00 : ffff8000102180e0: ldr x1, [x29, #144] : pte_modify(): : const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | : PTE_PROT_NONE | PTE_VALID | PTE_WRITE; : /* preserve the hardware dirty information */ : if (pte_hw_dirty(pte)) : pte = pte_mkdirty(pte); : pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); 0.00 : ffff8000102180e4: mov x0, #0xc1 // #193 : change_protection(): : { 0.00 : ffff8000102180e8: ldr x25, [x29, #128] 0.00 : ffff8000102180ec: mov x26, x28 0.00 : ffff8000102180f0: sub x1, x1, #0x1 0.00 : ffff8000102180f4: str x1, [x29, #136] : pte_modify(): 0.00 : ffff8000102180f8: ldr x1, [x29, #168] : change_protection(): 0.00 : ffff8000102180fc: mov x28, x25 : pte_modify(): 0.00 : ffff800010218100: movk x0, #0x468, lsl #48 : change_protection(): 0.00 : ffff800010218104: str xzr, [x29, #152] : pte_modify(): 0.00 : ffff800010218108: and x0, x1, x0 0.00 : ffff80001021810c: str x0, [x29, #176] : change_protection_range(): : next = pgd_addr_end(addr, end); 0.00 : ffff800010218110: mov x22, #0x8000000000 // #549755813888 0.00 : ffff800010218114: ldr x0, [x29, #160] 0.00 : ffff800010218118: add x22, x28, x22 0.00 : ffff80001021811c: ldr x2, [x29, #136] 0.00 : ffff800010218120: and x1, x22, #0xffffff8000000000 0.00 : ffff800010218124: str x1, [x29, #240] 0.00 : ffff800010218128: sub x1, x1, #0x1 0.00 : ffff80001021812c: ldr x0, [x0] 0.00 : ffff800010218130: cmp x1, x2 0.00 : ffff800010218134: b.cs ffff800010218740 // b.hs, b.nlast : pgd_none_or_clear_bad(): : : void pmd_clear_bad(pmd_t *); : : static inline int pgd_none_or_clear_bad(pgd_t *pgd) : { : if (pgd_none(*pgd)) 0.00 : ffff800010218138: cbz x0, ffff8000102184a0 : return 1; : if (unlikely(pgd_bad(*pgd))) { 0.00 : ffff80001021813c: tbz w0, #1, ffff800010218750 0.00 : ffff800010218140: ldr x1, [x29, #240] : change_pud_range(): : pud = pud_offset(p4d, addr); 0.00 : ffff800010218144: ubfx x21, x28, #30, #9 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010218148: ldr x0, [x29, #160] : change_pud_range(): 0.00 : ffff80001021814c: mov x25, x26 0.00 : ffff800010218150: sub x1, x1, #0x1 0.00 : ffff800010218154: str x1, [x29, #192] 0.00 : ffff800010218158: adrp x1, ffff8000112f2000 0.00 : ffff80001021815c: mov x26, x28 : __read_once_size(): 0.00 : ffff800010218160: ldr x0, [x0] : change_pud_range(): 0.00 : ffff800010218164: ldr x1, [x1, #1872] : pgd_page_paddr(): : return __pgd_to_phys(pgd); 0.00 : ffff800010218168: and x0, x0, #0xfffffffff000 : change_pud_range(): : unsigned long pages = 0; 0.00 : ffff80001021816c: str xzr, [x29, #224] : pud = pud_offset(p4d, addr); 0.00 : ffff800010218170: sub x0, x0, x1 0.00 : ffff800010218174: add x21, x0, x21, lsl #3 : next = pud_addr_end(addr, end); 0.00 : ffff800010218178: mov x8, #0x40000000 // #1073741824 0.00 : ffff80001021817c: add x8, x26, x8 0.00 : ffff800010218180: and x24, x8, #0xffffffffc0000000 0.00 : ffff800010218184: ldr x2, [x29, #192] 0.00 : ffff800010218188: sub x1, x24, #0x1 0.00 : ffff80001021818c: ldr x0, [x21] 0.00 : ffff800010218190: cmp x1, x2 0.00 : ffff800010218194: b.cs ffff80001021866c // b.hs, b.nlast : pud_none_or_clear_bad(): : return 0; : } : : static inline int pud_none_or_clear_bad(pud_t *pud) : { : if (pud_none(*pud)) 0.00 : ffff800010218198: cbz x0, ffff800010218498 : return 1; : if (unlikely(pud_bad(*pud))) { 0.00 : ffff80001021819c: tbz w0, #1, ffff800010218678 : change_pmd_range(): : pmd = pmd_offset(pud, addr); 0.00 : ffff8000102181a0: adrp x0, ffff8000112f2000 : range.start = 0; 0.00 : ffff8000102181a4: str xzr, [x29, #272] : pmd = pmd_offset(pud, addr); 0.00 : ffff8000102181a8: ubfx x7, x26, #21, #9 0.00 : ffff8000102181ac: str x21, [x29, #200] 0.00 : ffff8000102181b0: ldr x1, [x0, #1872] 0.00 : ffff8000102181b4: sub x20, x24, #0x1 : __read_once_size(): 0.00 : ffff8000102181b8: ldr x0, [x21] : change_pmd_range(): 0.00 : ffff8000102181bc: mov x21, x25 0.00 : ffff8000102181c0: mov x25, x26 : unsigned long nr_huge_updates = 0; 0.00 : ffff8000102181c4: str xzr, [x29, #232] : pud_page_paddr(): : return __pud_to_phys(pud); 0.00 : ffff8000102181c8: and x0, x0, #0xfffffffff000 : change_pmd_range(): : unsigned long pages = 0; 0.00 : ffff8000102181cc: str xzr, [x29, #248] : pmd = pmd_offset(pud, addr); 0.00 : ffff8000102181d0: sub x0, x0, x1 0.00 : ffff8000102181d4: add x23, x0, x7, lsl #3 0.00 : ffff8000102181d8: mov x26, x23 0.00 : ffff8000102181dc: nop : next = pmd_addr_end(addr, end); 0.00 : ffff8000102181e0: add x28, x25, #0x200, lsl #12 0.00 : ffff8000102181e4: ldr x0, [x26] 0.00 : ffff8000102181e8: and x28, x28, #0xffffffffffe00000 0.00 : ffff8000102181ec: sub x1, x28, #0x1 0.00 : ffff8000102181f0: cmp x1, x20 0.00 : ffff8000102181f4: csel x28, x28, x24, cc // cc = lo, ul, last : is_swap_pmd(): : extern spinlock_t *__pud_trans_huge_lock(pud_t *pud, : struct vm_area_struct *vma); : : static inline int is_swap_pmd(pmd_t pmd) : { : return !pmd_none(pmd) && !pmd_present(pmd); 0.00 : ffff8000102181f8: cbnz x0, ffff8000102182a4 : pmd_none_or_clear_bad_unless_trans_huge(): : if (pmd_none(pmdval)) 0.00 : ffff8000102181fc: cbz x0, ffff800010218230 : change_pmd_range(): : if (!range.start) { 0.00 : ffff800010218200: ldr x0, [x29, #272] 0.00 : ffff800010218204: cbnz x0, ffff800010218228 : mmu_notifier_range_init(&range, 0.00 : ffff800010218208: ldr x0, [x21, #64] : mmu_notifier_range_init(): : struct mm_struct *mm, : unsigned long start, : unsigned long end) : { : range->vma = vma; : range->event = event; 0.00 : ffff80001021820c: mov w1, #0x2 // #2 : range->mm = mm; 0.00 : ffff800010218210: stp x21, x0, [x29, #256] : range->start = start; : range->end = end; 0.00 : ffff800010218214: stp x25, x24, [x29, #272] : mmu_notifier_invalidate_range_start(): : if (mm_has_notifiers(range->mm)) { 0.00 : ffff800010218218: ldr x0, [x0, #816] : mmu_notifier_range_init(): : range->flags = flags; 0.00 : ffff80001021821c: str wzr, [x29, #288] : range->event = event; 0.00 : ffff800010218220: str w1, [x29, #292] : mmu_notifier_invalidate_range_start(): : if (mm_has_notifiers(range->mm)) { 0.00 : ffff800010218224: cbnz x0, ffff800010218614 0.00 : ffff800010218228: ldr x0, [x26] : is_swap_pmd(): 0.00 : ffff80001021822c: cbnz x0, ffff800010218630 : change_pmd_range(): : } while (pmd++, addr = next, addr != end); 0.00 : ffff800010218230: add x26, x26, #0x8 0.00 : ffff800010218234: mov x25, x28 0.00 : ffff800010218238: cmp x28, x24 0.00 : ffff80001021823c: b.ne ffff8000102181e0 // b.any : if (range.start) 0.00 : ffff800010218240: ldr x0, [x29, #272] 0.00 : ffff800010218244: mov x25, x21 0.00 : ffff800010218248: ldr x21, [x29, #200] 0.00 : ffff80001021824c: cbz x0, ffff80001021825c : mm_has_notifiers(): : return unlikely(mm->notifier_subscriptions); 0.00 : ffff800010218250: ldr x0, [x29, #264] : mmu_notifier_invalidate_range_end(): : if (mm_has_notifiers(range->mm)) 0.00 : ffff800010218254: ldr x0, [x0, #816] 0.00 : ffff800010218258: cbnz x0, ffff8000102187a4 : change_pmd_range(): : if (nr_huge_updates) 0.00 : ffff80001021825c: ldr x0, [x29, #232] 0.00 : ffff800010218260: cbnz x0, ffff8000102186b8 : change_pud_range(): : pages += change_pmd_range(vma, pud, addr, next, newprot, 0.00 : ffff800010218264: ldr x0, [x29, #224] 0.00 : ffff800010218268: mov x26, x28 0.00 : ffff80001021826c: ldr x1, [x29, #248] 0.00 : ffff800010218270: add x0, x0, x1 0.00 : ffff800010218274: str x0, [x29, #224] : } while (pud++, addr = next, addr != end); 0.00 : ffff800010218278: ldr x0, [x29, #240] 0.00 : ffff80001021827c: add x21, x21, #0x8 0.00 : ffff800010218280: cmp x0, x26 0.00 : ffff800010218284: b.ne ffff800010218178 // b.any : change_protection_range(): : pages += change_p4d_range(vma, pgd, addr, next, newprot, 0.00 : ffff800010218288: ldr x0, [x29, #152] 0.00 : ffff80001021828c: mov x26, x25 0.00 : ffff800010218290: ldr x1, [x29, #224] 0.00 : ffff800010218294: ldr x28, [x29, #240] 0.00 : ffff800010218298: add x0, x0, x1 0.00 : ffff80001021829c: str x0, [x29, #152] 0.00 : ffff8000102182a0: b ffff8000102184a4 : is_swap_pmd(): 0.00 : ffff8000102182a4: and x1, x0, #0x7ffffffffffffff 0.00 : ffff8000102182a8: and x1, x1, #0xfc00000000000001 0.00 : ffff8000102182ac: cbz x1, ffff8000102185ec : change_pmd_range(): : if (!is_swap_pmd(*pmd) && !pmd_devmap(*pmd) && 0.00 : ffff8000102182b0: tbz x0, #57, ffff8000102181fc : if (!range.start) { 0.00 : ffff8000102182b4: ldr x1, [x29, #272] 0.00 : ffff8000102182b8: cbz x1, ffff800010218208 : if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { 0.00 : ffff8000102182bc: tbz w0, #1, ffff8000102182c4 0.00 : ffff8000102182c0: tbz x0, #57, ffff8000102182ec : if (next - addr != HPAGE_PMD_SIZE) { 0.00 : ffff8000102182c4: sub x0, x28, x25 0.00 : ffff8000102182c8: cmp x0, #0x200, lsl #12 0.00 : ffff8000102182cc: b.eq ffff800010218648 // b.none : __split_huge_pmd(vma, pmd, addr, false, NULL); 0.00 : ffff8000102182d0: mov x0, x21 0.00 : ffff8000102182d4: mov x4, #0x0 // #0 0.00 : ffff8000102182d8: mov w3, #0x0 // #0 0.00 : ffff8000102182dc: mov x2, x25 0.00 : ffff8000102182e0: mov x1, x26 0.00 : ffff8000102182e4: bl ffff80001025da18 <__split_huge_pmd> 0.00 : ffff8000102182e8: ldr x0, [x26] : pmd_none_or_trans_huge_or_clear_bad(): : * : * pmd_none() is preseved for future condition checks on pmd migration : * entries and not confusing with this function name, although it is : * redundant with !pmd_present(). : */ : if (pmd_none(pmdval) || pmd_trans_huge(pmdval) || 0.00 : ffff8000102182ec: cbz x0, ffff800010218230 0.00 : ffff8000102182f0: tbz w0, #1, ffff800010218230 : change_pte_range(): : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 0.00 : ffff8000102182f4: adrp x1, ffff8000112f2000 : __read_once_size(): 0.00 : ffff8000102182f8: ldr x0, [x26] : pte_lockptr(): : } : #endif /* ALLOC_SPLIT_PTLOCKS */ : : static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) : { : return ptlock_ptr(pmd_page(*pmd)); 0.00 : ffff8000102182fc: ldr x22, [x26] : change_pte_range(): 0.00 : ffff800010218300: ubfx x3, x25, #12, #9 0.00 : ffff800010218304: ldr x2, [x1, #1872] : pte_lockptr(): 0.00 : ffff800010218308: adrp x1, ffff8000112f2000 : pmd_page_paddr(): : return __pmd_to_phys(pmd); 0.00 : ffff80001021830c: and x0, x0, #0xfffffffff000 : change_pte_range(): 0.00 : ffff800010218310: sub x0, x0, x2 : pte_lockptr(): 0.00 : ffff800010218314: ldr x1, [x1, #1880] : change_pte_range(): 0.00 : ffff800010218318: add x3, x0, x3, lsl #3 : pte_lockptr(): 0.00 : ffff80001021831c: ubfx x22, x22, #12, #36 0.00 : ffff800010218320: add x22, x1, x22, lsl #6 : spin_lock(): : raw_spin_lock_init(&(_lock)->rlock); \ : } while (0) : : static __always_inline void spin_lock(spinlock_t *lock) : { : raw_spin_lock(&lock->rlock); 0.00 : ffff800010218324: add x0, x22, #0x28 0.00 : ffff800010218328: stp x3, x0, [x29, #208] 0.00 : ffff80001021832c: bl ffff800010cd80f0 <_raw_spin_lock> : change_pte_range(): : int target_node = NUMA_NO_NODE; 0.00 : ffff800010218330: mov w0, #0xffffffff // #-1 0.00 : ffff800010218334: str w0, [x29, #188] : if (prot_numa && !(vma->vm_flags & VM_SHARED) && 0.00 : ffff800010218338: ldr x3, [x29, #208] 0.00 : ffff80001021833c: cbz w27, ffff800010218370 0.00 : ffff800010218340: ldr x0, [x21, #80] 0.00 : ffff800010218344: tbnz w0, #3, ffff800010218370 : atomic_read(&vma->vm_mm->mm_users) == 1) 0.00 : ffff800010218348: ldr x0, [x21, #64] : __read_once_size(): 0.00 : ffff80001021834c: ldr w0, [x0, #76] : change_pte_range(): : if (prot_numa && !(vma->vm_flags & VM_SHARED) && 0.00 : ffff800010218350: cmp w0, #0x1 0.00 : ffff800010218354: b.ne ffff800010218370 // b.any : numa_node_id(): : : #ifndef numa_node_id : /* Returns the number of the current Node. */ : static inline int numa_node_id(void) : { : return raw_cpu_read(numa_node); 0.00 : ffff800010218358: adrp x0, ffff800011526000 0.00 : ffff80001021835c: add x0, x0, #0xd58 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010218360: mrs x1, tpidr_el1 : numa_node_id(): 0.00 : ffff800010218364: ldr w0, [x0, x1] 0.00 : ffff800010218368: str w0, [x29, #188] 0.00 : ffff80001021836c: nop : change_pte_range(): : if (pte_present(oldpte)) { 0.00 : ffff800010218370: mov x22, #0x1 // #1 : pte_modify(): : pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); 0.00 : ffff800010218374: str x20, [x29, #208] 0.00 : ffff800010218378: mov x20, x21 0.00 : ffff80001021837c: mov x21, x25 0.00 : ffff800010218380: mov x25, x3 : change_pte_range(): : unsigned long pages = 0; 0.00 : ffff800010218384: mov x23, #0x0 // #0 : if (pte_present(oldpte)) { 0.00 : ffff800010218388: movk x22, #0x400, lsl #48 0.00 : ffff80001021838c: nop : oldpte = *pte; 0.00 : ffff800010218390: ldr x19, [x25] : if (pte_present(oldpte)) { 0.00 : ffff800010218394: ands x0, x19, x22 0.00 : ffff800010218398: b.eq ffff80001021851c // b.none : bool preserve_write = prot_numa && pte_write(oldpte); 0.00 : ffff80001021839c: cbnz w27, ffff800010218548 0.00 : ffff8000102183a0: mov w19, #0x0 // #0 : __xchg_case_64(): : } : : __XCHG_CASE(w, b, , 8, , , , , , ) : __XCHG_CASE(w, h, , 16, , , , , , ) : __XCHG_CASE(w, , , 32, , , , , , ) : __XCHG_CASE( , , , 64, , , , , , ) 0.00 : ffff8000102183a4: mov x1, #0x0 // #0 0.00 : ffff8000102183a8: prfm pstl1strm, [x25] 0.00 : ffff8000102183ac: ldxr x0, [x25] 0.00 : ffff8000102183b0: stxr w2, x1, [x25] 0.00 : ffff8000102183b4: cbnz w2, ffff8000102183ac : pte_modify(): : if (pte_hw_dirty(pte)) 0.00 : ffff8000102183b8: mov x2, #0x80 // #128 0.00 : ffff8000102183bc: mov x1, #0x8000000000000 // #2251799813685248 0.00 : ffff8000102183c0: movk x2, #0x8, lsl #48 0.00 : ffff8000102183c4: and x2, x0, x2 0.00 : ffff8000102183c8: cmp x2, x1 0.00 : ffff8000102183cc: b.ne ffff8000102183e4 // b.any : set_pte_bit(): : pte_val(pte) |= pgprot_val(prot); 0.00 : ffff8000102183d0: and x1, x0, #0xffffffffffffff7f 0.00 : ffff8000102183d4: tst x0, #0x8000000000000 0.00 : ffff8000102183d8: orr x1, x1, #0x80000000000000 0.00 : ffff8000102183dc: orr x0, x0, #0x80000000000000 0.00 : ffff8000102183e0: csel x0, x1, x0, ne // ne = any : pte_modify(): : pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); 0.00 : ffff8000102183e4: mov x1, #0xffffffffffffff3e // #-194 : clear_pte_bit(): : pte_val(pte) &= ~pgprot_val(prot); 0.00 : ffff8000102183e8: cmp w19, #0x0 : pte_modify(): : pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); 0.00 : ffff8000102183ec: movk x1, #0xfb97, lsl #48 0.00 : ffff8000102183f0: and x19, x0, x1 0.00 : ffff8000102183f4: ldr x0, [x29, #176] 0.00 : ffff8000102183f8: orr x19, x19, x0 : clear_pte_bit(): : pte_val(pte) &= ~pgprot_val(prot); 0.00 : ffff8000102183fc: and x0, x19, #0xffffffffffffff7f 0.00 : ffff800010218400: orr x0, x0, #0x8000000000000 0.00 : ffff800010218404: csel x19, x19, x0, eq // eq = none : change_pte_range(): : if (dirty_accountable && pte_dirty(ptent) && 0.00 : ffff800010218408: ldr w0, [x29, #184] 0.00 : ffff80001021840c: cbz w0, ffff800010218430 0.00 : ffff800010218410: tbnz x19, #55, ffff8000102186ac 0.00 : ffff800010218414: mov x0, #0x80 // #128 0.00 : ffff800010218418: movk x0, #0x8, lsl #48 0.00 : ffff80001021841c: and x1, x19, x0 0.00 : ffff800010218420: mov x0, #0x8000000000000 // #2251799813685248 0.00 : ffff800010218424: cmp x1, x0 0.00 : ffff800010218428: b.eq ffff8000102186ac // b.none 0.00 : ffff80001021842c: nop : set_pte_at(): : if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) 0.00 : ffff800010218430: tst x19, x22 0.00 : ffff800010218434: b.eq ffff800010218444 // b.none 0.00 : ffff800010218438: mov x0, #0x140000000000000 // #90071992547409920 0.00 : ffff80001021843c: tst x19, x0 0.00 : ffff800010218440: b.eq ffff800010218734 // b.none : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010218444: str x19, [x25] : set_pte(): : if (pte_valid_not_user(pte)) { 0.00 : ffff800010218448: mov x0, #0x41 // #65 0.00 : ffff80001021844c: and x19, x19, x0 0.00 : ffff800010218450: cmp x19, #0x1 0.00 : ffff800010218454: b.eq ffff80001021848c // b.none : change_pte_range(): : pages++; 0.00 : ffff800010218458: add x23, x23, #0x1 : } while (pte++, addr += PAGE_SIZE, addr != end); 0.00 : ffff80001021845c: add x21, x21, #0x1, lsl #12 0.00 : ffff800010218460: add x25, x25, #0x8 0.00 : ffff800010218464: cmp x28, x21 0.00 : ffff800010218468: b.ne ffff800010218390 // b.any 0.00 : ffff80001021846c: ldr x0, [x29, #248] 0.00 : ffff800010218470: mov x21, x20 0.00 : ffff800010218474: ldr x20, [x29, #208] 0.00 : ffff800010218478: add x0, x0, x23 0.00 : ffff80001021847c: str x0, [x29, #248] : spin_unlock(): : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ : } while (0) : : static __always_inline void spin_unlock(spinlock_t *lock) : { : raw_spin_unlock(&lock->rlock); 0.00 : ffff800010218480: ldr x0, [x29, #216] 0.00 : ffff800010218484: bl ffff800010cd7c38 <_raw_spin_unlock> 0.00 : ffff800010218488: b ffff800010218230 : set_pte(): : dsb(ishst); 0.00 : ffff80001021848c: dsb ishst : isb(); 0.00 : ffff800010218490: isb 0.00 : ffff800010218494: b ffff800010218458 : change_pud_range(): : next = pud_addr_end(addr, end); 0.00 : ffff800010218498: mov x26, x24 0.00 : ffff80001021849c: b ffff800010218278 : change_protection_range(): : next = pgd_addr_end(addr, end); 0.00 : ffff8000102184a0: ldr x28, [x29, #240] : } while (pgd++, addr = next, addr != end); 0.00 : ffff8000102184a4: ldr x0, [x29, #160] 0.00 : ffff8000102184a8: ldr x1, [x29, #240] 0.00 : ffff8000102184ac: add x0, x0, #0x8 0.00 : ffff8000102184b0: str x0, [x29, #160] 0.00 : ffff8000102184b4: ldr x0, [x29, #144] 0.00 : ffff8000102184b8: cmp x0, x1 0.00 : ffff8000102184bc: b.ne ffff800010218110 // b.any : if (pages) 0.00 : ffff8000102184c0: ldr x0, [x29, #152] 0.00 : ffff8000102184c4: cbnz x0, ffff8000102187b4 : arch_static_branch_jump(): 0.00 : ffff8000102184c8: b ffff800010218790 0.00 : ffff8000102184cc: b ffff800010218790 : __lse_atomic_sub(): : : #undef ATOMIC_FETCH_OP_AND : : static inline void __lse_atomic_sub(int i, atomic_t *v) : { : asm volatile( 0.00 : ffff8000102184d0: ldp x2, x1, [x29, #112] 0.00 : ffff8000102184d4: mov w0, #0x1 // #1 0.00 : ffff8000102184d8: add x1, x1, #0x34c 0.00 : ffff8000102184dc: neg w0, w0 0.00 : ffff8000102184e0: stadd w0, [x1] 0.00 : ffff8000102184e4: ldp x19, x20, [x29, #16] 0.00 : ffff8000102184e8: ldp x21, x22, [x29, #32] 0.00 : ffff8000102184ec: ldp x23, x24, [x29, #48] 0.00 : ffff8000102184f0: ldp x25, x26, [x29, #64] 0.00 : ffff8000102184f4: ldp x27, x28, [x29, #80] : change_protection(): : pages = hugetlb_change_protection(vma, start, end, newprot); : else : pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); : : return pages; : } 0.00 : ffff8000102184f8: adrp x0, ffff800011909000 0.00 : ffff8000102184fc: add x1, x0, #0x908 0.00 : ffff800010218500: ldr x0, [x29, #152] 0.00 : ffff800010218504: ldr x2, [x29, #296] 0.00 : ffff800010218508: ldr x1, [x1] 0.00 : ffff80001021850c: eor x1, x2, x1 0.00 : ffff800010218510: cbnz x1, ffff800010218874 0.00 : ffff800010218514: ldp x29, x30, [sp], #304 0.00 : ffff800010218518: ret : pte_to_swp_entry(): : swp_entry_t arch_entry; : : if (pte_swp_soft_dirty(pte)) : pte = pte_swp_clear_soft_dirty(pte); : arch_entry = __pte_to_swp_entry(pte); : return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); 0.00 : ffff80001021851c: lsr x0, x19, #2 0.00 : ffff800010218520: ubfx x19, x19, #8, #50 : swp_entry(): : ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK); 0.00 : ffff800010218524: orr x19, x19, x0, lsl #58 : swp_type(): : return (entry.val >> SWP_TYPE_SHIFT); 0.00 : ffff800010218528: lsr x0, x19, #58 : change_pte_range(): : if (is_write_migration_entry(entry)) { 0.00 : ffff80001021852c: cmp x0, #0x1f 0.00 : ffff800010218530: b.ne ffff80001021845c // b.any : swp_entry_to_pte(): : */ : static inline pte_t swp_entry_to_pte(swp_entry_t entry) : { : swp_entry_t arch_entry; : : arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 0.00 : ffff800010218534: lsl x19, x19, #8 : change_pte_range(): : pages++; 0.00 : ffff800010218538: add x23, x23, #0x1 : swp_entry_to_pte(): 0.00 : ffff80001021853c: orr x19, x19, #0x78 : __write_once_size(): 0.00 : ffff800010218540: str x19, [x25] 0.00 : ffff800010218544: b ffff80001021845c : change_pte_range(): : if (pte_protnone(oldpte)) 0.00 : ffff800010218548: mov x1, #0x400000000000000 // #288230376151711744 0.00 : ffff80001021854c: cmp x0, x1 0.00 : ffff800010218550: b.eq ffff80001021845c // b.none : page = vm_normal_page(vma, addr, oldpte); 0.00 : ffff800010218554: mov x1, x21 0.00 : ffff800010218558: mov x2, x19 0.00 : ffff80001021855c: mov x0, x20 0.00 : ffff800010218560: bl ffff800010209c98 0.00 : ffff800010218564: mov x1, x0 : if (!page || PageKsm(page)) 0.00 : ffff800010218568: cbz x0, ffff80001021845c : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001021856c: ldr x0, [x0, #8] : compound_head(): : static inline struct page *compound_head(struct page *page) : { : unsigned long head = READ_ONCE(page->compound_head); : : if (unlikely(head & 1)) : return (struct page *) (head - 1); 0.00 : ffff800010218570: tst x0, #0x1 0.00 : ffff800010218574: sub x0, x0, #0x1 0.00 : ffff800010218578: csel x0, x0, x1, ne // ne = any : PageKsm(): : * anon_vma, but to that page's node of the stable tree. : */ : static __always_inline int PageKsm(struct page *page) : { : page = compound_head(page); : return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == 0.00 : ffff80001021857c: ldr x0, [x0, #24] 0.00 : ffff800010218580: and x0, x0, #0x3 : change_pte_range(): 0.00 : ffff800010218584: cmp x0, #0x3 0.00 : ffff800010218588: b.eq ffff80001021845c // b.none : is_cow_mapping(): : */ : #define page_order_unsafe(page) READ_ONCE(page_private(page)) : : static inline bool is_cow_mapping(vm_flags_t flags) : { : return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 0.00 : ffff80001021858c: ldr x0, [x20, #80] 0.00 : ffff800010218590: mov x2, #0x28 // #40 0.00 : ffff800010218594: and x0, x0, x2 : change_pte_range(): : if (is_cow_mapping(vma->vm_flags) && 0.00 : ffff800010218598: cmp x0, #0x20 0.00 : ffff80001021859c: b.eq ffff80001021870c // b.none : __read_once_size(): 0.00 : ffff8000102185a0: ldr x0, [x1, #8] : compound_head(): : if (unlikely(head & 1)) 0.00 : ffff8000102185a4: tst x0, #0x1 0.00 : ffff8000102185a8: sub x0, x0, #0x1 0.00 : ffff8000102185ac: csel x0, x0, x1, ne // ne = any : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff8000102185b0: ldr x0, [x0] : change_pte_range(): : if (page_is_file_cache(page) && PageDirty(page)) 0.00 : ffff8000102185b4: tbnz w0, #19, ffff8000102185d0 : __read_once_size(): 0.00 : ffff8000102185b8: ldr x0, [x1, #8] : compound_head(): : return (struct page *) (head - 1); 0.00 : ffff8000102185bc: tst x0, #0x1 0.00 : ffff8000102185c0: sub x0, x0, #0x1 0.00 : ffff8000102185c4: csel x0, x0, x1, ne // ne = any : test_bit(): 0.00 : ffff8000102185c8: ldr x0, [x0] : change_pte_range(): 0.00 : ffff8000102185cc: tbnz w0, #3, ffff80001021845c : page_to_nid(): : return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK; 0.00 : ffff8000102185d0: ldr x0, [x1] : change_pte_range(): : if (target_node == page_to_nid(page)) 0.00 : ffff8000102185d4: ldr w1, [x29, #188] : page_to_nid(): 0.00 : ffff8000102185d8: lsr x0, x0, #62 : change_pte_range(): 0.00 : ffff8000102185dc: cmp w1, w0 0.00 : ffff8000102185e0: b.eq ffff80001021845c // b.none : bool preserve_write = prot_numa && pte_write(oldpte); 0.00 : ffff8000102185e4: ubfx x19, x19, #51, #1 0.00 : ffff8000102185e8: b ffff8000102183a4 : change_pmd_range(): : if (!range.start) { 0.00 : ffff8000102185ec: ldr x0, [x29, #272] 0.00 : ffff8000102185f0: cbnz x0, ffff8000102182c4 : mmu_notifier_range_init(&range, 0.00 : ffff8000102185f4: ldr x0, [x21, #64] : mmu_notifier_range_init(): : range->event = event; 0.00 : ffff8000102185f8: mov w1, #0x2 // #2 : range->mm = mm; 0.00 : ffff8000102185fc: stp x21, x0, [x29, #256] : range->end = end; 0.00 : ffff800010218600: stp x25, x24, [x29, #272] : mmu_notifier_invalidate_range_start(): : if (mm_has_notifiers(range->mm)) { 0.00 : ffff800010218604: ldr x0, [x0, #816] : mmu_notifier_range_init(): : range->flags = flags; 0.00 : ffff800010218608: str wzr, [x29, #288] : range->event = event; 0.00 : ffff80001021860c: str w1, [x29, #292] : mmu_notifier_invalidate_range_start(): : if (mm_has_notifiers(range->mm)) { 0.00 : ffff800010218610: cbz x0, ffff800010218228 : range->flags |= MMU_NOTIFIER_RANGE_BLOCKABLE; 0.00 : ffff800010218614: mov w0, #0x1 // #1 0.00 : ffff800010218618: str w0, [x29, #288] : __mmu_notifier_invalidate_range_start(range); 0.00 : ffff80001021861c: add x0, x29, #0x100 0.00 : ffff800010218620: bl ffff8000102476d8 <__mmu_notifier_invalidate_range_start> 0.00 : ffff800010218624: ldr x0, [x26] : is_swap_pmd(): 0.00 : ffff800010218628: cbz x0, ffff800010218230 0.00 : ffff80001021862c: nop 0.00 : ffff800010218630: and x1, x0, #0x7ffffffffffffff 0.00 : ffff800010218634: and x1, x1, #0xfc00000000000001 0.00 : ffff800010218638: cbnz x1, ffff8000102182bc : change_pmd_range(): : if (next - addr != HPAGE_PMD_SIZE) { 0.00 : ffff80001021863c: sub x0, x28, x25 0.00 : ffff800010218640: cmp x0, #0x200, lsl #12 0.00 : ffff800010218644: b.ne ffff8000102182d0 // b.any : int nr_ptes = change_huge_pmd(vma, pmd, addr, 0.00 : ffff800010218648: ldr x3, [x29, #168] 0.00 : ffff80001021864c: mov w4, w27 0.00 : ffff800010218650: mov x2, x25 0.00 : ffff800010218654: mov x1, x26 0.00 : ffff800010218658: mov x0, x21 0.00 : ffff80001021865c: bl ffff80001025d800 : if (nr_ptes) { 0.00 : ffff800010218660: cbnz w0, ffff800010218688 0.00 : ffff800010218664: ldr x0, [x26] 0.00 : ffff800010218668: b ffff8000102182ec : pud_none_or_clear_bad(): : if (pud_none(*pud)) 0.00 : ffff80001021866c: cbz x0, ffff800010218288 0.00 : ffff800010218670: ldr x24, [x29, #240] : if (unlikely(pud_bad(*pud))) { 0.00 : ffff800010218674: tbnz w0, #1, ffff8000102181a0 : pud_clear_bad(pud); 0.00 : ffff800010218678: mov x26, x24 0.00 : ffff80001021867c: mov x0, x21 0.00 : ffff800010218680: bl ffff80001021b228 0.00 : ffff800010218684: b ffff800010218278 : change_pmd_range(): : if (nr_ptes == HPAGE_PMD_NR) { 0.00 : ffff800010218688: cmp w0, #0x200 0.00 : ffff80001021868c: b.ne ffff800010218230 // b.any : pages += HPAGE_PMD_NR; 0.00 : ffff800010218690: ldr x0, [x29, #248] 0.00 : ffff800010218694: add x0, x0, #0x200 0.00 : ffff800010218698: str x0, [x29, #248] : nr_huge_updates++; 0.00 : ffff80001021869c: ldr x0, [x29, #232] 0.00 : ffff8000102186a0: add x0, x0, #0x1 0.00 : ffff8000102186a4: str x0, [x29, #232] 0.00 : ffff8000102186a8: b ffff800010218230 : clear_pte_bit(): : pte_val(pte) &= ~pgprot_val(prot); 0.00 : ffff8000102186ac: and x19, x19, #0xffffffffffffff7f 0.00 : ffff8000102186b0: orr x19, x19, #0x8000000000000 0.00 : ffff8000102186b4: b ffff800010218430 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000102186b8: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000102186bc: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000102186c0: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102186c4: str w0, [x1, #16] : count_vm_events(): : raw_cpu_add(vm_event_states.event[item], delta); : } : : static inline void count_vm_events(enum vm_event_item item, long delta) : { : this_cpu_add(vm_event_states.event[item], delta); 0.00 : ffff8000102186c8: adrp x0, ffff800011526000 0.00 : ffff8000102186cc: add x0, x0, #0xb00 : __my_cpu_offset(): 0.00 : ffff8000102186d0: mrs x2, tpidr_el1 : __percpu_add_case_64(): : : PERCPU_RW_OPS(8) : PERCPU_RW_OPS(16) : PERCPU_RW_OPS(32) : PERCPU_RW_OPS(64) : PERCPU_OP(add, add, stadd) 0.00 : ffff8000102186d4: add x0, x0, x2 0.00 : ffff8000102186d8: ldr x2, [x29, #232] 0.00 : ffff8000102186dc: ldxr x4, [x0] 0.00 : ffff8000102186e0: add x4, x4, x2 0.00 : ffff8000102186e4: stxr w3, x4, [x0] 0.00 : ffff8000102186e8: cbnz w3, ffff8000102186dc : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102186ec: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000102186f0: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000102186f4: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000102186f8: cbz x0, ffff800010218704 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000102186fc: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff800010218700: cbnz x0, ffff800010218264 : count_vm_events(): 0.00 : ffff800010218704: bl ffff800010cd2e78 0.00 : ffff800010218708: b ffff800010218264 : test_bit(): 0.00 : ffff80001021870c: ldr x0, [x1] : PageCompound(): : return test_bit(PG_head, &page->flags) || PageTail(page); 0.00 : ffff800010218710: tbz w0, #16, ffff800010218760 : page_mapcount(): : return __page_mapcount(page); 0.00 : ffff800010218714: mov x0, x1 0.00 : ffff800010218718: str x1, [x29, #104] 0.00 : ffff80001021871c: bl ffff8000101f2ed0 <__page_mapcount> 0.00 : ffff800010218720: ldr x1, [x29, #104] 0.00 : ffff800010218724: nop : change_pte_range(): : if (is_cow_mapping(vma->vm_flags) && 0.00 : ffff800010218728: cmp w0, #0x1 0.00 : ffff80001021872c: b.eq ffff8000102185a0 // b.none 0.00 : ffff800010218730: b ffff80001021845c : set_pte_at(): : __sync_icache_dcache(pte); 0.00 : ffff800010218734: mov x0, x19 0.00 : ffff800010218738: bl ffff8000100a3198 <__sync_icache_dcache> 0.00 : ffff80001021873c: b ffff800010218444 : pgd_none_or_clear_bad(): : if (pgd_none(*pgd)) 0.00 : ffff800010218740: cbz x0, ffff8000102184c0 0.00 : ffff800010218744: ldr x1, [x29, #144] 0.00 : ffff800010218748: str x1, [x29, #240] : if (unlikely(pgd_bad(*pgd))) { 0.00 : ffff80001021874c: tbnz w0, #1, ffff800010218140 : pgd_clear_bad(pgd); 0.00 : ffff800010218750: ldr x0, [x29, #160] 0.00 : ffff800010218754: ldr x28, [x29, #240] 0.00 : ffff800010218758: bl ffff80001021b1c0 0.00 : ffff80001021875c: b ffff8000102184a4 : __read_once_size(): 0.00 : ffff800010218760: ldr x0, [x1, #8] : PageCompound(): 0.00 : ffff800010218764: tbnz w0, #0, ffff800010218714 : __read_once_size(): 0.00 : ffff800010218768: ldr w0, [x1, #48] : page_mapcount(): : return atomic_read(&page->_mapcount) + 1; 0.00 : ffff80001021876c: add w0, w0, #0x1 0.00 : ffff800010218770: b ffff800010218728 : change_protection(): : pages = hugetlb_change_protection(vma, start, end, newprot); 0.00 : ffff800010218774: bl ffff800010240e18 0.00 : ffff800010218778: str x0, [x29, #152] 0.00 : ffff80001021877c: b ffff8000102184f8 : __ll_sc_atomic_add(): : ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) 0.00 : ffff800010218780: ldr x0, [x29, #120] 0.00 : ffff800010218784: add x2, x0, #0x34c 0.00 : ffff800010218788: b ffff800010218d5c <__arm64_sys_mprotect+0x234> 0.00 : ffff80001021878c: b ffff8000102180e0 : __ll_sc_atomic_sub(): : ATOMIC_OPS(sub, sub, J) 0.00 : ffff800010218790: ldr x1, [x29, #120] 0.00 : ffff800010218794: mov w0, #0x1 // #1 0.00 : ffff800010218798: add x3, x1, #0x34c 0.00 : ffff80001021879c: b ffff800010218d74 <__arm64_sys_mprotect+0x24c> 0.00 : ffff8000102187a0: b ffff8000102184e4 : mmu_notifier_invalidate_range_end(): : __mmu_notifier_invalidate_range_end(range, false); 0.00 : ffff8000102187a4: mov w1, #0x0 // #0 0.00 : ffff8000102187a8: add x0, x29, #0x100 0.00 : ffff8000102187ac: bl ffff8000102478e8 <__mmu_notifier_invalidate_range_end> 0.00 : ffff8000102187b0: b ffff80001021825c : __flush_tlb_range(): : { : unsigned long asid = ASID(vma->vm_mm); : unsigned long addr; : : start = round_down(start, stride); : end = round_up(end, stride); 0.00 : ffff8000102187b4: ldp x1, x0, [x29, #128] : : if ((end - start) >= (MAX_TLBI_OPS * stride)) { 0.00 : ffff8000102187b8: mov x3, #0x1fffff // #2097151 : unsigned long asid = ASID(vma->vm_mm); 0.00 : ffff8000102187bc: ldr x2, [x26, #64] : end = round_up(end, stride); 0.00 : ffff8000102187c0: orr x0, x0, #0xfff : start = round_down(start, stride); 0.00 : ffff8000102187c4: and x1, x1, #0xfffffffffffff000 : end = round_up(end, stride); 0.00 : ffff8000102187c8: add x0, x0, #0x1 0.00 : ffff8000102187cc: ldr x2, [x2, #736] : if ((end - start) >= (MAX_TLBI_OPS * stride)) { 0.00 : ffff8000102187d0: sub x4, x0, x1 0.00 : ffff8000102187d4: cmp x4, x3 0.00 : ffff8000102187d8: lsl x2, x2, #48 0.00 : ffff8000102187dc: b.hi ffff800010218840 // b.pmore : } : : /* Convert the stride into units of 4k */ : stride >>= 12; : : start = __TLBI_VADDR(start, asid); 0.00 : ffff8000102187e0: ubfx x1, x1, #12, #44 : end = __TLBI_VADDR(end, asid); 0.00 : ffff8000102187e4: ubfx x0, x0, #12, #44 : start = __TLBI_VADDR(start, asid); 0.00 : ffff8000102187e8: orr x1, x2, x1 : end = __TLBI_VADDR(end, asid); 0.00 : ffff8000102187ec: orr x0, x2, x0 : : dsb(ishst); 0.00 : ffff8000102187f0: dsb ishst : for (addr = start; addr < end; addr += stride) { 0.00 : ffff8000102187f4: cmp x1, x0 0.00 : ffff8000102187f8: b.cs ffff800010218834 // b.hs, b.nlast : arm64_kernel_unmapped_at_el0(): : : extern bool arm64_use_ng_mappings; : : static inline bool arm64_kernel_unmapped_at_el0(void) : { : return arm64_use_ng_mappings; 0.00 : ffff8000102187fc: adrp x2, ffff800011af2000 0.00 : ffff800010218800: ldrb w2, [x2, #2168] 0.00 : ffff800010218804: nop : __flush_tlb_range(): : if (last_level) { : __tlbi(vale1is, addr); : __tlbi_user(vale1is, addr); : } else { : __tlbi(vae1is, addr); 0.00 : ffff800010218808: tlbi vae1is, x1 100.00 : ffff80001021880c: nop 0.00 : ffff800010218810: nop : __tlbi_user(vae1is, addr); 0.00 : ffff800010218814: cbz w2, ffff800010218828 0.00 : ffff800010218818: orr x3, x1, #0x1000000000000 0.00 : ffff80001021881c: tlbi vae1is, x3 0.00 : ffff800010218820: nop 0.00 : ffff800010218824: nop : for (addr = start; addr < end; addr += stride) { 0.00 : ffff800010218828: add x1, x1, #0x1 0.00 : ffff80001021882c: cmp x0, x1 0.00 : ffff800010218830: b.ne ffff800010218808 // b.any : } : } : dsb(ish); 0.00 : ffff800010218834: dsb ish 0.00 : ffff800010218838: b ffff8000102184c8 : change_protection_range(): : BUG_ON(addr >= end); 0.00 : ffff80001021883c: brk #0x800 : flush_tlb_mm(): : dsb(ishst); 0.00 : ffff800010218840: dsb ishst : __tlbi(aside1is, asid); 0.00 : ffff800010218844: tlbi aside1is, x2 0.00 : ffff800010218848: nop 0.00 : ffff80001021884c: nop : arm64_kernel_unmapped_at_el0(): 0.00 : ffff800010218850: adrp x0, ffff800011af2000 : flush_tlb_mm(): : __tlbi_user(aside1is, asid); 0.00 : ffff800010218854: ldrb w0, [x0, #2168] 0.00 : ffff800010218858: cbz w0, ffff800010218834 0.00 : ffff80001021885c: orr x0, x2, #0x1000000000000 0.00 : ffff800010218860: tlbi aside1is, x0 0.00 : ffff800010218864: nop 0.00 : ffff800010218868: nop : __flush_tlb_range(): : dsb(ish); 0.00 : ffff80001021886c: dsb ish 0.00 : ffff800010218870: b ffff8000102184c8 0.00 : ffff800010218874: stp x19, x20, [x29, #16] 0.00 : ffff800010218878: stp x21, x22, [x29, #32] 0.00 : ffff80001021887c: stp x23, x24, [x29, #48] 0.00 : ffff800010218880: stp x25, x26, [x29, #64] 0.00 : ffff800010218884: stp x27, x28, [x29, #80] : change_protection(): : } 0.00 : ffff800010218888: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016d070 : : timekeeping_update(): : tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); : } : : /* must hold timekeeper_lock */ : static void timekeeping_update(struct timekeeper *tk, unsigned int action) : { 0.00 : ffff80001016d070: stp x29, x30, [sp, #-48]! 0.00 : ffff80001016d074: mov x29, sp 0.00 : ffff80001016d078: stp x19, x20, [sp, #16] 0.00 : ffff80001016d07c: mov x19, x0 0.00 : ffff80001016d080: stp x21, x22, [sp, #32] 0.00 : ffff80001016d084: mov w20, w1 : if (action & TK_CLEAR_NTP) { 0.00 : ffff80001016d088: tbnz w20, #0, ffff80001016d188 : tk_update_leap_state(): : tk->next_leap_ktime = ntp_get_next_leap(); 0.00 : ffff80001016d08c: bl ffff80001016f148 0.00 : ffff80001016d090: str x0, [x19, #184] : if (tk->next_leap_ktime != KTIME_MAX) 0.00 : ffff80001016d094: mov x1, #0x7fffffffffffffff // #9223372036854775807 0.00 : ffff80001016d098: cmp x0, x1 0.00 : ffff80001016d09c: b.eq ffff80001016d0ac // b.none : tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real); 0.00 : ffff80001016d0a0: ldr x1, [x19, #144] 0.00 : ffff80001016d0a4: sub x0, x0, x1 0.00 : ffff80001016d0a8: str x0, [x19, #184] : tk_update_ktime_data(): : nsec = (u32) tk->wall_to_monotonic.tv_nsec; 0.00 : ffff80001016d0ac: ldp x3, x5, [x19, #128] : tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 0.00 : ffff80001016d0b0: mov x4, #0xca00 // #51712 : seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); 0.00 : ffff80001016d0b4: ldr x0, [x19, #112] : tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 0.00 : ffff80001016d0b8: movk x4, #0x3b9a, lsl #16 : nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 0.00 : ffff80001016d0bc: ldr w7, [x19, #28] : if (nsec >= NSEC_PER_SEC) 0.00 : ffff80001016d0c0: mov w6, #0xc9ff // #51711 : nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 0.00 : ffff80001016d0c4: ldr x1, [x19, #32] : seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); 0.00 : ffff80001016d0c8: add x0, x0, x3 : tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); 0.00 : ffff80001016d0cc: ldr x2, [x19, #192] : tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 0.00 : ffff80001016d0d0: and x3, x5, #0xffffffff : if (nsec >= NSEC_PER_SEC) 0.00 : ffff80001016d0d4: movk w6, #0x3b9a, lsl #16 : update_pvclock_gtod(): : raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); 0.00 : ffff80001016d0d8: adrp x21, ffff800011b20000 <__log_buf+0x1fc10> : tk_update_ktime_data(): : nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 100.00 : ffff80001016d0dc: lsr x1, x1, x7 0.00 : ffff80001016d0e0: add w1, w1, w5 : seconds++; 0.00 : ffff80001016d0e4: cmp w1, w6 : tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 0.00 : ffff80001016d0e8: madd x3, x0, x4, x3 : tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); 0.00 : ffff80001016d0ec: mul x1, x2, x4 : seconds++; 0.00 : ffff80001016d0f0: cinc x0, x0, hi // hi = pmore : tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 0.00 : ffff80001016d0f4: str x3, [x19, #40] : timekeeping_update(): : : tk_update_leap_state(tk); : tk_update_ktime_data(tk); : : update_vsyscall(tk); : update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); 0.00 : ffff80001016d0f8: and w22, w20, #0x4 : tk_update_ktime_data(): : tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); 0.00 : ffff80001016d0fc: str x1, [x19, #96] : tk->ktime_sec = seconds; 0.00 : ffff80001016d100: str x0, [x19, #120] : timekeeping_update(): : update_vsyscall(tk); 0.00 : ffff80001016d104: mov x0, x19 0.00 : ffff80001016d108: bl ffff80001017c6b0 : update_pvclock_gtod(): : raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); 0.00 : ffff80001016d10c: add x0, x21, #0xec0 0.00 : ffff80001016d110: mov x2, x19 0.00 : ffff80001016d114: ubfx x1, x20, #2, #1 0.00 : ffff80001016d118: add x0, x0, #0x168 0.00 : ffff80001016d11c: bl ffff80001010e828 : timekeeping_update(): : : tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real; 0.00 : ffff80001016d120: ldr x0, [x19, #40] : update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); 0.00 : ffff80001016d124: adrp x6, ffff800011925000 : tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real; 0.00 : ffff80001016d128: ldr x1, [x19, #144] : update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); 0.00 : ffff80001016d12c: add x6, x6, #0x400 : tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real; 0.00 : ffff80001016d130: add x0, x0, x1 0.00 : ffff80001016d134: str x0, [x19, #48] : update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); 0.00 : ffff80001016d138: add x1, x6, #0x80 0.00 : ffff80001016d13c: mov x0, x19 0.00 : ffff80001016d140: bl ffff80001016cff8 : update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw); 0.00 : ffff80001016d144: mov x1, x6 0.00 : ffff80001016d148: add x0, x19, #0x38 0.00 : ffff80001016d14c: bl ffff80001016cff8 : : if (action & TK_CLOCK_WAS_SET) 0.00 : ffff80001016d150: cbz w22, ffff80001016d160 : tk->clock_was_set_seq++; 0.00 : ffff80001016d154: ldr w0, [x19, #172] 0.00 : ffff80001016d158: add w0, w0, #0x1 0.00 : ffff80001016d15c: str w0, [x19, #172] : /* : * The mirroring of the data to the shadow-timekeeper needs : * to happen last here to ensure we don't over-write the : * timekeeper structure on the next update with stale data : */ : if (action & TK_MIRROR) 0.00 : ffff80001016d160: tbz w20, #1, ffff80001016d178 : memcpy(&shadow_timekeeper, &tk_core.timekeeper, 0.00 : ffff80001016d164: add x0, x21, #0xec0 0.00 : ffff80001016d168: mov x2, #0x118 // #280 0.00 : ffff80001016d16c: add x1, x0, #0x48 0.00 : ffff80001016d170: add x0, x0, #0x170 0.00 : ffff80001016d174: bl ffff800010cb7dc0 <__memcpy> : sizeof(tk_core.timekeeper)); : } 0.00 : ffff80001016d178: ldp x19, x20, [sp, #16] 0.00 : ffff80001016d17c: ldp x21, x22, [sp, #32] 0.00 : ffff80001016d180: ldp x29, x30, [sp], #48 0.00 : ffff80001016d184: ret : tk->ntp_error = 0; 0.00 : ffff80001016d188: str xzr, [x0, #256] : ntp_clear(); 0.00 : ffff80001016d18c: bl ffff80001016f0e0 0.00 : ffff80001016d190: b ffff80001016d08c Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010160c28 : : rcu_core_si(): : do_nocb_deferred_wakeup(rdp); : trace_rcu_utilization(TPS("End RCU core")); : } : : static void rcu_core_si(struct softirq_action *h) : { 0.00 : ffff800010160c28: stp x29, x30, [sp, #-16]! 0.00 : ffff800010160c2c: mov x29, sp : rcu_core(); 0.00 : ffff800010160c30: bl ffff800010160760 : } 100.00 : ffff800010160c34: ldp x29, x30, [sp], #16 0.00 : ffff800010160c38: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015cbb8 : : rcu_advance_cbs(): : * Returns true if the RCU grace-period kthread needs to be awakened. : * : * The caller must hold rnp->lock with interrupts disabled. : */ : static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) : { 100.00 : ffff80001015cbb8: stp x29, x30, [sp, #-48]! 0.00 : ffff80001015cbbc: mov x29, sp 0.00 : ffff80001015cbc0: stp x19, x20, [sp, #16] : rcu_lockdep_assert_cblist_protected(rdp); : raw_lockdep_assert_held_rcu_node(rnp); : : /* If no pending (not yet ready to invoke) callbacks, nothing to do. */ : if (!rcu_segcblist_pend_cbs(&rdp->cblist)) 0.00 : ffff80001015cbc4: add x20, x1, #0x50 : { 0.00 : ffff80001015cbc8: str x21, [sp, #32] 0.00 : ffff80001015cbcc: mov x19, x1 0.00 : ffff80001015cbd0: mov x21, x0 : if (!rcu_segcblist_pend_cbs(&rdp->cblist)) 0.00 : ffff80001015cbd4: mov x0, x20 0.00 : ffff80001015cbd8: bl ffff800010162bb0 0.00 : ffff80001015cbdc: tst w0, #0xff 0.00 : ffff80001015cbe0: b.ne ffff80001015cbf8 // b.any : return false; 0.00 : ffff80001015cbe4: mov w0, #0x0 // #0 : */ : rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); : : /* Classify any remaining callbacks. */ : return rcu_accelerate_cbs(rnp, rdp); : } 0.00 : ffff80001015cbe8: ldr x21, [sp, #32] 0.00 : ffff80001015cbec: ldp x19, x20, [sp, #16] 0.00 : ffff80001015cbf0: ldp x29, x30, [sp], #48 0.00 : ffff80001015cbf4: ret : rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); 0.00 : ffff80001015cbf8: ldr x1, [x21, #8] 0.00 : ffff80001015cbfc: mov x0, x20 0.00 : ffff80001015cc00: bl ffff800010162e58 : return rcu_accelerate_cbs(rnp, rdp); 0.00 : ffff80001015cc04: mov x1, x19 0.00 : ffff80001015cc08: mov x0, x21 0.00 : ffff80001015cc0c: bl ffff80001015c9d8 0.00 : ffff80001015cc10: and w0, w0, #0xff : } 0.00 : ffff80001015cc14: ldp x19, x20, [sp, #16] 0.00 : ffff80001015cc18: ldr x21, [sp, #32] 0.00 : ffff80001015cc1c: ldp x29, x30, [sp], #48 0.00 : ffff80001015cc20: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010137e40 : : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 100.00 : ffff800010137e40: nop : housekeeping_cpumask(): : const struct cpumask *housekeeping_cpumask(enum hk_flags flags) : { : if (static_branch_unlikely(&housekeeping_overridden)) : if (housekeeping_flags & flags) : return housekeeping_mask; : return cpu_possible_mask; 0.00 : ffff800010137e44: adrp x0, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010137e48: add x0, x0, #0x1a0 : } 0.00 : ffff800010137e4c: ret : if (housekeeping_flags & flags) 0.00 : ffff800010137e50: adrp x3, ffff800011afd000 0.00 : ffff800010137e54: add x2, x3, #0x2c0 : return housekeeping_mask; 0.00 : ffff800010137e58: add x2, x2, #0x8 0.00 : ffff800010137e5c: adrp x1, ffff80001190a000 <__per_cpu_offset+0x6d8> : if (housekeeping_flags & flags) 0.00 : ffff800010137e60: ldr w3, [x3, #704] : return housekeeping_mask; 0.00 : ffff800010137e64: add x1, x1, #0x1a0 0.00 : ffff800010137e68: tst w0, w3 0.00 : ffff800010137e6c: csel x0, x1, x2, eq // eq = none : } 0.00 : ffff800010137e70: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010116fe8 : : default_wake_function(): : exception_exit(prev_state); : } : : int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, : void *key) : { 100.00 : ffff800010116fe8: stp x29, x30, [sp, #-16]! 0.00 : ffff800010116fec: mov x29, sp : return try_to_wake_up(curr->private, mode, wake_flags); 0.00 : ffff800010116ff0: ldr x0, [x0, #8] 0.00 : ffff800010116ff4: bl ffff800010116ad0 : } 0.00 : ffff800010116ff8: ldp x29, x30, [sp], #16 0.00 : ffff800010116ffc: ret Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010116f00 : : wake_up_process(): : * Return: 1 if the process was woken up, 0 if it was already running. : * : * This function executes a full memory barrier before accessing the task state. : */ : int wake_up_process(struct task_struct *p) : { 100.00 : ffff800010116f00: stp x29, x30, [sp, #-16]! : return try_to_wake_up(p, TASK_NORMAL, 0); 0.00 : ffff800010116f04: mov w2, #0x0 // #0 0.00 : ffff800010116f08: mov w1, #0x3 // #3 : { 0.00 : ffff800010116f0c: mov x29, sp : return try_to_wake_up(p, TASK_NORMAL, 0); 0.00 : ffff800010116f10: bl ffff800010116ad0 : } 0.00 : ffff800010116f14: ldp x29, x30, [sp], #16 0.00 : ffff800010116f18: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001016aa20 : : update_process_times(): : /* : * Called from the timer interrupt handler to charge one tick to the current : * process. user_tick is 1 if the tick is user time, 0 for system. : */ : void update_process_times(int user_tick) : { 0.00 : ffff80001016aa20: stp x29, x30, [sp, #-32]! : struct task_struct *p = current; : : /* Note: this timer irq context must be accounted for as well. */ : account_process_tick(p, user_tick); 0.00 : ffff80001016aa24: mov w1, w0 : { 0.00 : ffff80001016aa28: mov x29, sp 0.00 : ffff80001016aa2c: stp x19, x20, [sp, #16] 0.00 : ffff80001016aa30: mov w20, w0 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001016aa34: mrs x19, sp_el0 : update_process_times(): : account_process_tick(p, user_tick); 0.00 : ffff80001016aa38: mov x0, x19 0.00 : ffff80001016aa3c: bl ffff80001011b948 : run_local_timers(); 100.00 : ffff80001016aa40: bl ffff80001016a9c8 : rcu_sched_clock_irq(user_tick); 0.00 : ffff80001016aa44: mov w0, w20 0.00 : ffff80001016aa48: bl ffff800010161288 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001016aa4c: ldr w0, [x19, #16] : update_process_times(): : #ifdef CONFIG_IRQ_WORK : if (in_irq()) 0.00 : ffff80001016aa50: tst w0, #0xf0000 0.00 : ffff80001016aa54: b.eq ffff80001016aa5c // b.none : irq_work_tick(); 0.00 : ffff80001016aa58: bl ffff8000101b4898 : #endif : scheduler_tick(); 0.00 : ffff80001016aa5c: bl ffff800010118200 : if (IS_ENABLED(CONFIG_POSIX_TIMERS)) : run_posix_cpu_timers(); 0.00 : ffff80001016aa60: bl ffff800010176168 : } 0.00 : ffff80001016aa64: ldp x19, x20, [sp, #16] 0.00 : ffff80001016aa68: ldp x29, x30, [sp], #32 0.00 : ffff80001016aa6c: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001015c9d8 : : rcu_accelerate_cbs(): : * the RCU grace-period kthread. : * : * The caller must hold rnp->lock with interrupts disabled. : */ : static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) : { 100.00 : ffff80001015c9d8: stp x29, x30, [sp, #-80]! 0.00 : ffff80001015c9dc: mov x29, sp 0.00 : ffff80001015c9e0: str x19, [sp, #16] : : rcu_lockdep_assert_cblist_protected(rdp); : raw_lockdep_assert_held_rcu_node(rnp); : : /* If no pending (not yet ready to invoke) callbacks, nothing to do. */ : if (!rcu_segcblist_pend_cbs(&rdp->cblist)) 0.00 : ffff80001015c9e4: add x19, x1, #0x50 : { 0.00 : ffff80001015c9e8: stp x21, x22, [sp, #32] 0.00 : ffff80001015c9ec: mov x22, x0 0.00 : ffff80001015c9f0: str x23, [sp, #48] : if (!rcu_segcblist_pend_cbs(&rdp->cblist)) 0.00 : ffff80001015c9f4: mov x0, x19 : { 0.00 : ffff80001015c9f8: mov x23, x1 : return false; 0.00 : ffff80001015c9fc: mov w21, #0x0 // #0 : if (!rcu_segcblist_pend_cbs(&rdp->cblist)) 0.00 : ffff80001015ca00: bl ffff800010162bb0 0.00 : ffff80001015ca04: tst w0, #0xff 0.00 : ffff80001015ca08: b.ne ffff80001015ca24 // b.any : if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) : trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccWaitCB")); : else : trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccReadyCB")); : return ret; : } 0.00 : ffff80001015ca0c: mov w0, w21 0.00 : ffff80001015ca10: ldr x19, [sp, #16] 0.00 : ffff80001015ca14: ldp x21, x22, [sp, #32] 0.00 : ffff80001015ca18: ldr x23, [sp, #48] 0.00 : ffff80001015ca1c: ldp x29, x30, [sp], #80 0.00 : ffff80001015ca20: ret 0.00 : ffff80001015ca24: str x24, [x29, #56] : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff80001015ca28: adrp x24, ffff800011922000 0.00 : ffff80001015ca2c: add x0, x24, #0xc80 0.00 : ffff80001015ca30: str x20, [x29, #24] 0.00 : ffff80001015ca34: ldr x20, [x0, #8776] : rcu_seq_snap(): : */ : static inline unsigned long rcu_seq_snap(unsigned long *sp) : { : unsigned long s; : : s = (READ_ONCE(*sp) + 2 * RCU_SEQ_STATE_MASK + 1) & ~RCU_SEQ_STATE_MASK; 0.00 : ffff80001015ca38: add x20, x20, #0x7 0.00 : ffff80001015ca3c: and x20, x20, #0xfffffffffffffffc : smp_mb(); /* Above access must not bleed into critical section. */ 0.00 : ffff80001015ca40: dmb ish : rcu_accelerate_cbs(): : if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) 0.00 : ffff80001015ca44: mov x1, x20 0.00 : ffff80001015ca48: mov x0, x19 0.00 : ffff80001015ca4c: bl ffff800010162ed8 0.00 : ffff80001015ca50: tst w0, #0xff 0.00 : ffff80001015ca54: b.ne ffff80001015ca84 // b.any : bool ret = false; 0.00 : ffff80001015ca58: mov w21, #0x0 // #0 : __read_once_size(): 0.00 : ffff80001015ca5c: ldr x0, [x23, #96] : rcu_accelerate_cbs(): 0.00 : ffff80001015ca60: ldr x20, [x29, #24] 0.00 : ffff80001015ca64: ldr x24, [x29, #56] : __read_once_size(): 0.00 : ffff80001015ca68: ldr x0, [x0] : rcu_accelerate_cbs(): : } 0.00 : ffff80001015ca6c: mov w0, w21 0.00 : ffff80001015ca70: ldr x19, [sp, #16] 0.00 : ffff80001015ca74: ldp x21, x22, [sp, #32] 0.00 : ffff80001015ca78: ldr x23, [sp, #48] 0.00 : ffff80001015ca7c: ldp x29, x30, [sp], #80 0.00 : ffff80001015ca80: ret 0.00 : ffff80001015ca84: str x25, [x29, #64] 0.00 : ffff80001015ca88: mov x19, x22 : rcu_seq_started(): : * Given a snapshot from rcu_seq_snap(), determine whether or not the : * corresponding update-side operation has started. : */ : static inline bool rcu_seq_started(unsigned long *sp, unsigned long s) : { : return ULONG_CMP_LT((s - 1) & ~RCU_SEQ_STATE_MASK, READ_ONCE(*sp)); 0.00 : ffff80001015ca8c: sub x25, x20, #0x4 : rcu_start_this_gp(): : if (ULONG_CMP_GE(rnp->gp_seq_needed, gp_seq_req) || 0.00 : ffff80001015ca90: ldr x1, [x19, #16] 0.00 : ffff80001015ca94: sub x0, x1, x20 0.00 : ffff80001015ca98: tbz x0, #63, ffff80001015cb5c : __read_once_size(): 0.00 : ffff80001015ca9c: ldr x0, [x19, #8] : rcu_seq_started(): 0.00 : ffff80001015caa0: sub x0, x25, x0 : rcu_start_this_gp(): 0.00 : ffff80001015caa4: tbnz x0, #63, ffff80001015cb5c : rcu_seq_started(&rnp->gp_seq, gp_seq_req) || 0.00 : ffff80001015caa8: cmp x22, x19 0.00 : ffff80001015caac: b.eq ffff80001015cb44 // b.none : __read_once_size(): 0.00 : ffff80001015cab0: ldr x0, [x19, #8] : rcu_start_this_gp(): : (rnp != rnp_start && 0.00 : ffff80001015cab4: tst x0, #0x3 0.00 : ffff80001015cab8: b.ne ffff80001015cb80 // b.any : __read_once_size(): 0.00 : ffff80001015cabc: ldr x0, [x19, #8] : rcu_start_this_gp(): : rnp->gp_seq_needed = gp_seq_req; 0.00 : ffff80001015cac0: str x20, [x19, #16] : if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) { 0.00 : ffff80001015cac4: tst x0, #0x3 0.00 : ffff80001015cac8: b.ne ffff80001015cbac // b.any : if (rnp != rnp_start && rnp->parent != NULL) 0.00 : ffff80001015cacc: ldr x0, [x19, #120] 0.00 : ffff80001015cad0: cbz x0, ffff80001015cafc : raw_spin_unlock_rcu_node(rnp); 0.00 : ffff80001015cad4: mov x0, x19 0.00 : ffff80001015cad8: bl ffff800010cd7c38 <_raw_spin_unlock> : if (!rnp->parent) 0.00 : ffff80001015cadc: ldr x21, [x19, #120] 0.00 : ffff80001015cae0: cbz x21, ffff80001015cafc : if (rnp != rnp_start) 0.00 : ffff80001015cae4: cmp x22, x21 0.00 : ffff80001015cae8: b.eq ffff80001015caf4 // b.none : raw_spin_lock_rcu_node(rnp); 0.00 : ffff80001015caec: mov x0, x21 0.00 : ffff80001015caf0: bl ffff800010cd80f0 <_raw_spin_lock> : rcu_accelerate_cbs(): : { 0.00 : ffff80001015caf4: mov x19, x21 0.00 : ffff80001015caf8: b ffff80001015ca90 : __read_once_size(): 0.00 : ffff80001015cafc: add x24, x24, #0xc80 0.00 : ffff80001015cb00: ldr x0, [x24, #8776] : rcu_start_this_gp(): : if (rcu_gp_in_progress()) { 0.00 : ffff80001015cb04: tst x0, #0x3 0.00 : ffff80001015cb08: b.ne ffff80001015cba0 // b.any : WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT); 0.00 : ffff80001015cb0c: add x1, x24, #0x2, lsl #12 : rcu_state.gp_req_activity = jiffies; 0.00 : ffff80001015cb10: adrp x3, ffff800011907000 : if (!rcu_state.gp_kthread) { 0.00 : ffff80001015cb14: ldr x2, [x24, #8784] : WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT); 0.00 : ffff80001015cb18: ldrh w0, [x1, #624] 0.00 : ffff80001015cb1c: orr w0, w0, #0x1 : __write_once_size(): : : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; 0.00 : ffff80001015cb20: and w0, w0, #0xffff 0.00 : ffff80001015cb24: strh w0, [x1, #624] : rcu_start_this_gp(): : rcu_state.gp_req_activity = jiffies; 0.00 : ffff80001015cb28: ldr x0, [x3, #2432] 0.00 : ffff80001015cb2c: str x0, [x24, #9080] : if (!rcu_state.gp_kthread) { 0.00 : ffff80001015cb30: cbz x2, ffff80001015cba0 : ret = true; /* Caller must wake GP kthread. */ 0.00 : ffff80001015cb34: mov w21, #0x1 // #1 0.00 : ffff80001015cb38: ldr x1, [x19, #16] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001015cb3c: ldr x0, [x24, #8776] 0.00 : ffff80001015cb40: b ffff80001015cb60 0.00 : ffff80001015cb44: ldr x0, [x22, #8] : rcu_start_this_gp(): : rnp->gp_seq_needed = gp_seq_req; 0.00 : ffff80001015cb48: str x20, [x22, #16] : if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) { 0.00 : ffff80001015cb4c: tst x0, #0x3 0.00 : ffff80001015cb50: b.eq ffff80001015cadc // b.none 0.00 : ffff80001015cb54: ldr x25, [x29, #64] 0.00 : ffff80001015cb58: b ffff80001015ca58 : bool ret = false; 0.00 : ffff80001015cb5c: mov w21, #0x0 // #0 : if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) { 0.00 : ffff80001015cb60: sub x20, x20, x1 0.00 : ffff80001015cb64: tbnz x20, #63, ffff80001015cb90 : if (rnp != rnp_start) 0.00 : ffff80001015cb68: cmp x22, x19 0.00 : ffff80001015cb6c: b.eq ffff80001015cb78 // b.none : raw_spin_unlock_rcu_node(rnp); 0.00 : ffff80001015cb70: mov x0, x19 0.00 : ffff80001015cb74: bl ffff800010cd7c38 <_raw_spin_unlock> 0.00 : ffff80001015cb78: ldr x25, [x29, #64] 0.00 : ffff80001015cb7c: b ffff80001015ca5c : if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) { 0.00 : ffff80001015cb80: sub x20, x20, x1 0.00 : ffff80001015cb84: tbz x20, #63, ffff80001015cbac : bool ret = false; 0.00 : ffff80001015cb88: mov w21, #0x0 // #0 0.00 : ffff80001015cb8c: nop : rnp_start->gp_seq_needed = rnp->gp_seq_needed; 0.00 : ffff80001015cb90: str x1, [x22, #16] : rdp->gp_seq_needed = rnp->gp_seq_needed; 0.00 : ffff80001015cb94: ldr x0, [x19, #16] 0.00 : ffff80001015cb98: str x0, [x23, #8] 0.00 : ffff80001015cb9c: b ffff80001015cb68 : bool ret = false; 0.00 : ffff80001015cba0: mov w21, #0x0 // #0 0.00 : ffff80001015cba4: ldr x1, [x19, #16] 0.00 : ffff80001015cba8: b ffff80001015cb60 0.00 : ffff80001015cbac: mov w21, #0x0 // #0 0.00 : ffff80001015cbb0: b ffff80001015cb70 Percent | Source code & Disassembly of vmlinux for cycles (2 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010120020 : : dequeue_task_fair(): : * The dequeue_task method is called before nr_running is : * decreased. We remove the task from the rbtree and : * update the fair scheduling stats: : */ : static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) : { 0.00 : ffff800010120020: stp x29, x30, [sp, #-128]! 0.00 : ffff800010120024: mov x29, sp 0.00 : ffff800010120028: stp x21, x22, [sp, #32] 0.00 : ffff80001012002c: mov x22, x0 0.00 : ffff800010120030: str x23, [sp, #48] : struct cfs_rq *cfs_rq; : struct sched_entity *se = &p->se; : int task_sleep = flags & DEQUEUE_SLEEP; 0.00 : ffff800010120034: and w0, w2, #0x1 : { 0.00 : ffff800010120038: str x25, [sp, #64] 0.00 : ffff80001012003c: mov x23, x1 : int task_sleep = flags & DEQUEUE_SLEEP; 0.00 : ffff800010120040: str w0, [x29, #100] : idle_policy(): : */ : #define RUNTIME_INF ((u64)~0ULL) : : static inline int idle_policy(int policy) : { : return policy == SCHED_IDLE; 0.00 : ffff800010120044: ldr w3, [x1, #728] : dequeue_task_fair(): : int idle_h_nr_running = task_has_idle_policy(p); : bool was_sched_idle = sched_idle_rq(rq); 0.00 : ffff800010120048: ldr w0, [x22, #4] : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff80001012004c: ldr w1, [x22, #160] : idle_policy(): 0.00 : ffff800010120050: cmp w3, #0x5 0.00 : ffff800010120054: cset w21, eq // eq = none : sched_idle_rq(): 0.00 : ffff800010120058: cmp w1, w0 0.00 : ffff80001012005c: cset w25, eq // eq = none 0.00 : ffff800010120060: cmp w0, #0x0 0.00 : ffff800010120064: csel w1, w25, wzr, ne // ne = any 0.00 : ffff800010120068: str w1, [x29, #96] : dequeue_task_fair(): : : for_each_sched_entity(se) { 0.00 : ffff80001012006c: adds x25, x23, #0xc0 0.00 : ffff800010120070: b.eq ffff800010120464 // b.none 0.00 : ffff800010120074: stp x19, x20, [x29, #16] 0.00 : ffff800010120078: mov w20, w2 0.00 : ffff80001012007c: str x24, [x29, #56] : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff800010120080: adrp x24, ffff800011afb000 0.00 : ffff800010120084: stp x26, x27, [x29, #72] 0.00 : ffff800010120088: add x0, x24, #0xd80 0.00 : ffff80001012008c: str x28, [x29, #88] 0.00 : ffff800010120090: str x0, [x29, #104] 0.00 : ffff800010120094: nop : dequeue_task_fair(): : cfs_rq = cfs_rq_of(se); 0.00 : ffff800010120098: ldr x19, [x25, #128] : dequeue_entity(): : update_curr(cfs_rq); 0.00 : ffff80001012009c: mov x0, x19 0.00 : ffff8000101200a0: bl ffff80001011fab8 : cfs_rq_clock_pelt(): : return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; : } : #else : static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) : { : return rq_clock_pelt(rq_of(cfs_rq)); 0.00 : ffff8000101200a4: ldr x0, [x19, #304] : update_load_avg(): : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 0.00 : ffff8000101200a8: ldr x1, [x25, #192] : rq_clock_pelt(): : return rq->clock_pelt - rq->lost_idle_time; 0.00 : ffff8000101200ac: ldr x4, [x0, #2440] 0.00 : ffff8000101200b0: ldr x0, [x0, #2448] 0.00 : ffff8000101200b4: sub x27, x4, x0 : update_load_avg(): 0.00 : ffff8000101200b8: cbz x1, ffff8000101200cc : __update_load_avg_se(now, cfs_rq, se); 0.00 : ffff8000101200bc: mov x2, x25 0.00 : ffff8000101200c0: mov x1, x19 0.00 : ffff8000101200c4: mov x0, x27 0.00 : ffff8000101200c8: bl ffff800010134868 <__update_load_avg_se> : update_cfs_rq_load_avg(): : if (cfs_rq->removed.nr) { 0.00 : ffff8000101200cc: ldr w28, [x19, #196] 0.00 : ffff8000101200d0: cbz w28, ffff80001012017c : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff8000101200d4: ldr w5, [x19, #156] 0.00 : ffff8000101200d8: mov w3, #0xb67e // #46718 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff8000101200dc: add x28, x19, #0xc0 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff8000101200e0: add w24, w5, w3 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff8000101200e4: mov x0, x28 0.00 : ffff8000101200e8: bl ffff800010cd80f0 <_raw_spin_lock> : cfs_rq->removed.nr = 0; 0.00 : ffff8000101200ec: str wzr, [x19, #196] : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff8000101200f0: ldp x8, x26, [x19, #200] 0.00 : ffff8000101200f4: stp xzr, xzr, [x19, #200] : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff8000101200f8: ldr x10, [x19, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff8000101200fc: mov x0, x28 : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010120100: str xzr, [x19, #216] 0.00 : ffff800010120104: stp x8, x10, [x29, #112] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff800010120108: bl ffff800010cd7c38 <_raw_spin_unlock> : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff80001012010c: ldp x8, x10, [x29, #112] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010120110: mov w0, w24 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010120114: ldr x2, [x19, #160] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010120118: mov x9, #0x1 // #1 : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff80001012011c: ldr x6, [x19, #272] : update_cfs_rq_load_avg(): : decayed = 1; 0.00 : ffff800010120120: mov w28, w9 : sub_positive(&sa->load_avg, r); 0.00 : ffff800010120124: sub x1, x2, x8 : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010120128: str x9, [x19, #264] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff80001012012c: cmp x2, x1 0.00 : ffff800010120130: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120134: str x1, [x19, #160] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120138: sub x1, x6, x10 0.00 : ffff80001012013c: str x1, [x19, #272] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120140: ldr x6, [x19, #136] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010120144: msub x0, x0, x8, x6 0.00 : ffff800010120148: cmp x6, x0 0.00 : ffff80001012014c: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120150: str x0, [x19, #136] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120154: ldr x1, [x19, #176] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_avg, r); 0.00 : ffff800010120158: sub x0, x1, x26 0.00 : ffff80001012015c: cmp x1, x0 0.00 : ffff800010120160: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120164: str x0, [x19, #176] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120168: ldr w1, [x19, #152] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_sum, r * divider); 0.00 : ffff80001012016c: msub w0, w24, w26, w1 0.00 : ffff800010120170: cmp w1, w0 0.00 : ffff800010120174: csel w0, w0, wzr, cs // cs = hs, nlast : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010120178: str w0, [x19, #152] : update_cfs_rq_load_avg(): : decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 0.00 : ffff80001012017c: mov x1, x19 0.00 : ffff800010120180: mov x0, x27 0.00 : ffff800010120184: bl ffff800010134a68 <__update_load_avg_cfs_rq> 0.00 : ffff800010120188: orr w0, w0, w28 : propagate_entity_load_avg(): : if (entity_is_task(se)) 0.00 : ffff80001012018c: ldr x1, [x25, #136] 0.00 : ffff800010120190: cbz x1, ffff8000101204e8 : if (!gcfs_rq->propagate) 0.00 : ffff800010120194: ldr x2, [x1, #264] 0.00 : ffff800010120198: cbz x2, ffff8000101204e8 : gcfs_rq->propagate = 0; 0.00 : ffff80001012019c: str xzr, [x1, #264] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff8000101201a0: mov x5, #0x1 // #1 : propagate_entity_load_avg(): : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 0.00 : ffff8000101201a4: ldr x2, [x1, #272] : cfs_rq = cfs_rq_of(se); 0.00 : ffff8000101201a8: ldr x4, [x25, #128] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff8000101201ac: ldr x0, [x4, #272] 0.00 : ffff8000101201b0: add x0, x0, x2 0.00 : ffff8000101201b4: stp x5, x0, [x4, #264] : propagate_entity_load_avg(): : update_tg_cfs_util(cfs_rq, se, gcfs_rq); 0.00 : ffff8000101201b8: ldr x2, [x1, #176] : update_tg_cfs_util(): : long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 0.00 : ffff8000101201bc: ldr x0, [x25, #240] 0.00 : ffff8000101201c0: sub x0, x2, x0 : if (!delta) 0.00 : ffff8000101201c4: cmp x0, #0x0 0.00 : ffff8000101201c8: b.eq ffff8000101201fc // b.none : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff8000101201cc: mov w5, #0xba7e // #47742 : se->avg.util_avg = gcfs_rq->avg.util_avg; 0.00 : ffff8000101201d0: str x2, [x25, #240] : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff8000101201d4: mul w2, w2, w5 0.00 : ffff8000101201d8: str w2, [x25, #216] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101201dc: ldr x2, [x4, #176] : update_tg_cfs_util(): : add_positive(&cfs_rq->avg.util_avg, delta); 0.00 : ffff8000101201e0: add x0, x0, x2 0.00 : ffff8000101201e4: ccmp x2, x0, #0x2, lt // lt = tstop 0.00 : ffff8000101201e8: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101201ec: str x0, [x4, #176] : update_tg_cfs_util(): : cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff8000101201f0: ldr x0, [x4, #176] 0.00 : ffff8000101201f4: mul w5, w5, w0 0.00 : ffff8000101201f8: str w5, [x4, #152] : update_tg_cfs_runnable(): : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 0.00 : ffff8000101201fc: ldr x0, [x1, #272] : if (!runnable_sum) 0.00 : ffff800010120200: cmp x0, #0x0 0.00 : ffff800010120204: b.eq ffff800010120320 // b.none : gcfs_rq->prop_runnable_sum = 0; 0.00 : ffff800010120208: str xzr, [x1, #272] : if (runnable_sum >= 0) { 0.00 : ffff80001012020c: b.lt ffff80001012055c // b.tstop : runnable_sum += se->avg.load_sum; 0.00 : ffff800010120210: ldr x2, [x25, #200] : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff800010120214: mov x1, #0xba7e // #47742 : runnable_sum += se->avg.load_sum; 0.00 : ffff800010120218: add x0, x0, x2 : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff80001012021c: cmp x0, x1 0.00 : ffff800010120220: csel x1, x0, x1, le : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff800010120224: ldr w5, [x25, #216] : div_s64_rem(): : * Return: sets ``*remainder``, then returns dividend / divisor : */ : static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) : { : *remainder = dividend % divisor; : return dividend / divisor; 0.00 : ffff800010120228: mov x3, #0x5041 // #20545 : se_weight(): : /* : * XXX we want to get rid of these helpers and use the full load resolution. : */ : static inline long se_weight(struct sched_entity *se) : { : return scale_load_down(se->load.weight); 0.00 : ffff80001012022c: ldr x0, [x25] : div_s64_rem(): 0.00 : ffff800010120230: movk x3, #0xafb, lsl #16 : update_tg_cfs_runnable(): : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff800010120234: ldr x6, [x25, #224] : div_s64_rem(): 0.00 : ffff800010120238: movk x3, #0x400, lsl #32 : update_tg_cfs_runnable(): : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff80001012023c: lsr w5, w5, #10 : div_s64_rem(): 0.00 : ffff800010120240: movk x3, #0xafb5, lsl #48 : update_tg_cfs_runnable(): : runnable_sum = max(runnable_sum, running_sum); 0.00 : ffff800010120244: cmp x5, x1 : se_weight(): 0.00 : ffff800010120248: lsr x0, x0, #10 : update_tg_cfs_runnable(): 0.00 : ffff80001012024c: csel x5, x5, x1, ge // ge = tcont : se->avg.load_sum = runnable_sum; 0.00 : ffff800010120250: str x5, [x25, #200] : load_sum = (s64)se_weight(se) * runnable_sum; 0.00 : ffff800010120254: mul x1, x0, x5 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 0.00 : ffff800010120258: msub x2, x2, x0, x1 : div_s64_rem(): 0.00 : ffff80001012025c: lsr x0, x1, #1 0.00 : ffff800010120260: mov x1, #0x5041 // #20545 0.00 : ffff800010120264: movk x1, #0xafb, lsl #16 0.00 : ffff800010120268: movk x1, #0x400, lsl #32 0.00 : ffff80001012026c: movk x1, #0xafb5, lsl #48 0.00 : ffff800010120270: umulh x0, x0, x1 0.00 : ffff800010120274: lsr x0, x0, #14 : update_tg_cfs_runnable(): : se->avg.load_avg = load_avg; 0.00 : ffff800010120278: str x0, [x25, #224] : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff80001012027c: sub x0, x0, x6 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120280: ldr x1, [x4, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_avg, delta_avg); 0.00 : ffff800010120284: cmp x0, #0x0 0.00 : ffff800010120288: add x0, x0, x1 0.00 : ffff80001012028c: ccmp x1, x0, #0x2, lt // lt = tstop 0.00 : ffff800010120290: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120294: str x0, [x4, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_sum, delta_sum); 0.00 : ffff800010120298: cmp x2, #0x0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012029c: ldr x1, [x4, #136] : update_tg_cfs_runnable(): 0.00 : ffff8000101202a0: add x0, x2, x1 0.00 : ffff8000101202a4: ccmp x1, x0, #0x2, lt // lt = tstop 0.00 : ffff8000101202a8: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101202ac: str x0, [x4, #136] : se_runnable(): : } : : static inline long se_runnable(struct sched_entity *se) : { : return scale_load_down(se->runnable_weight); 0.00 : ffff8000101202b0: ldr x1, [x25, #16] : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff8000101202b4: ldr w2, [x25, #64] : se_runnable(): 0.00 : ffff8000101202b8: lsr x1, x1, #10 : update_tg_cfs_runnable(): : runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 0.00 : ffff8000101202bc: mul x1, x1, x5 : div_s64_rem(): 0.00 : ffff8000101202c0: lsr x0, x1, #1 0.00 : ffff8000101202c4: umulh x0, x0, x3 0.00 : ffff8000101202c8: lsr x0, x0, #14 : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff8000101202cc: cbz w2, ffff800010120318 : delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 0.00 : ffff8000101202d0: ldr x7, [x25, #232] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101202d4: ldr x8, [x4, #168] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff8000101202d8: ldr x2, [x25] : update_tg_cfs_runnable(): 0.00 : ffff8000101202dc: sub x7, x0, x7 : se_weight(se) * se->avg.runnable_load_sum; 0.00 : ffff8000101202e0: ldr x9, [x25, #208] : add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 0.00 : ffff8000101202e4: add x6, x7, x8 0.00 : ffff8000101202e8: cmp x7, #0x0 0.00 : ffff8000101202ec: ccmp x8, x6, #0x2, lt // lt = tstop : se_weight(): 0.00 : ffff8000101202f0: lsr x2, x2, #10 : update_tg_cfs_runnable(): 0.00 : ffff8000101202f4: csel x6, x6, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101202f8: str x6, [x4, #168] : update_tg_cfs_runnable(): : delta_sum = runnable_load_sum - 0.00 : ffff8000101202fc: msub x1, x2, x9, x1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120300: ldr x2, [x4, #144] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 0.00 : ffff800010120304: cmp x1, #0x0 0.00 : ffff800010120308: add x1, x1, x2 0.00 : ffff80001012030c: ccmp x2, x1, #0x2, lt // lt = tstop 0.00 : ffff800010120310: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120314: str x1, [x4, #144] : update_tg_cfs_runnable(): : se->avg.runnable_load_sum = runnable_sum; 0.00 : ffff800010120318: str x5, [x25, #208] : se->avg.runnable_load_avg = runnable_load_avg; 0.00 : ffff80001012031c: str x0, [x25, #232] : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010120320: ldr x1, [x19, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff800010120324: add x0, x1, #0x80 0.00 : ffff800010120328: cmp x19, x0 0.00 : ffff80001012032c: b.eq ffff8000101204fc // b.none : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff800010120330: ldr x0, [x19, #336] 0.00 : ffff800010120334: ldr x1, [x29, #104] 0.00 : ffff800010120338: cmp x0, x1 0.00 : ffff80001012033c: b.eq ffff800010120370 // b.none : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 0.00 : ffff800010120340: ldr x1, [x19, #160] 0.00 : ffff800010120344: ldr x2, [x19, #256] : if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 0.00 : ffff800010120348: subs x1, x1, x2 0.00 : ffff80001012034c: cneg x4, x1, mi // mi = first 0.00 : ffff800010120350: cmp x4, x2, lsr #6 0.00 : ffff800010120354: b.ls ffff800010120370 // b.plast : atomic_long_add(delta, &cfs_rq->tg->load_avg); 0.00 : ffff800010120358: add x2, x0, #0x140 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001012035c: b ffff800010120548 0.00 : ffff800010120360: b ffff800010120548 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff800010120364: stadd x1, [x2] : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff800010120368: ldr x0, [x19, #160] 0.00 : ffff80001012036c: str x0, [x19, #256] : dequeue_runnable_load_avg(): : cfs_rq->runnable_weight -= se->runnable_weight; 0.00 : ffff800010120370: ldr x2, [x25, #16] : dequeue_entity(): : clear_buddies(cfs_rq, se); 0.00 : ffff800010120374: mov x1, x25 : dequeue_runnable_load_avg(): : cfs_rq->runnable_weight -= se->runnable_weight; 0.00 : ffff800010120378: ldr x0, [x19, #16] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012037c: ldr x4, [x19, #168] : dequeue_runnable_load_avg(): 0.00 : ffff800010120380: sub x0, x0, x2 0.00 : ffff800010120384: str x0, [x19, #16] : dequeue_entity(): : clear_buddies(cfs_rq, se); 0.00 : ffff800010120388: mov x0, x19 : dequeue_runnable_load_avg(): : sub_positive(&cfs_rq->avg.runnable_load_avg, se->avg.runnable_load_avg); 0.00 : ffff80001012038c: ldr x2, [x25, #232] 0.00 : ffff800010120390: sub x2, x4, x2 0.00 : ffff800010120394: cmp x4, x2 0.00 : ffff800010120398: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012039c: str x2, [x19, #168] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff8000101203a0: ldr x2, [x25, #16] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101203a4: ldr x4, [x19, #144] : dequeue_runnable_load_avg(): : sub_positive(&cfs_rq->avg.runnable_load_sum, 0.00 : ffff8000101203a8: ldr x5, [x25, #208] : se_runnable(): 0.00 : ffff8000101203ac: lsr x2, x2, #10 : dequeue_runnable_load_avg(): 0.00 : ffff8000101203b0: msub x2, x2, x5, x4 0.00 : ffff8000101203b4: cmp x4, x2 0.00 : ffff8000101203b8: csel x2, x2, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101203bc: str x2, [x19, #144] : dequeue_entity(): : clear_buddies(cfs_rq, se); 0.00 : ffff8000101203c0: bl ffff80001011cfb0 : if (se != cfs_rq->curr) 0.00 : ffff8000101203c4: ldr x0, [x19, #72] 0.00 : ffff8000101203c8: cmp x25, x0 0.00 : ffff8000101203cc: b.eq ffff8000101203ec // b.none : rb_erase_cached(): : } : : static inline void rb_erase_cached(struct rb_node *node, : struct rb_root_cached *root) : { : if (root->rb_leftmost == node) 0.00 : ffff8000101203d0: ldr x0, [x19, #64] : __dequeue_entity(): : rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline); 0.00 : ffff8000101203d4: add x28, x25, #0x18 0.00 : ffff8000101203d8: add x1, x19, #0x38 : rb_erase_cached(): 0.00 : ffff8000101203dc: cmp x28, x0 0.00 : ffff8000101203e0: b.eq ffff800010120530 // b.none : root->rb_leftmost = rb_next(node); : rb_erase(node, &root->rb_root); 0.00 : ffff8000101203e4: mov x0, x28 0.00 : ffff8000101203e8: bl ffff800010cc4750 : dequeue_entity(): : se->on_rq = 0; 0.00 : ffff8000101203ec: str wzr, [x25, #64] : account_entity_dequeue(cfs_rq, se); 0.00 : ffff8000101203f0: mov x1, x25 0.00 : ffff8000101203f4: mov x0, x19 0.00 : ffff8000101203f8: bl ffff80001011c5c8 : if (!(flags & DEQUEUE_SLEEP)) 0.00 : ffff8000101203fc: tbnz w20, #0, ffff800010120410 : se->vruntime -= cfs_rq->min_vruntime; 0.00 : ffff800010120400: ldr x1, [x19, #48] 0.00 : ffff800010120404: ldr x0, [x25, #88] 0.00 : ffff800010120408: sub x0, x0, x1 0.00 : ffff80001012040c: str x0, [x25, #88] : update_cfs_group(se); 0.00 : ffff800010120410: mov x0, x25 0.00 : ffff800010120414: bl ffff80001011fd78 : if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) 0.00 : ffff800010120418: and w0, w20, #0x6 0.00 : ffff80001012041c: cmp w0, #0x2 0.00 : ffff800010120420: b.eq ffff80001012042c // b.none : update_min_vruntime(cfs_rq); 0.00 : ffff800010120424: mov x0, x19 0.00 : ffff800010120428: bl ffff80001011c438 : dequeue_task_fair(): : * post the final h_nr_running decrement below. : */ : if (cfs_rq_throttled(cfs_rq)) : break; : cfs_rq->h_nr_running--; : cfs_rq->idle_h_nr_running -= idle_h_nr_running; 52.03 : ffff80001012042c: ldp w1, w0, [x19, #28] : : /* Don't dequeue parent if it has other entities besides us */ : if (cfs_rq->load.weight) { 0.00 : ffff800010120430: ldr x2, [x19] : cfs_rq->h_nr_running--; 0.00 : ffff800010120434: sub w1, w1, #0x1 : cfs_rq->idle_h_nr_running -= idle_h_nr_running; 0.00 : ffff800010120438: sub w0, w0, w21 0.00 : ffff80001012043c: stp w1, w0, [x19, #28] : if (cfs_rq->load.weight) { 0.00 : ffff800010120440: cbnz x2, ffff800010120588 : for_each_sched_entity(se) { 0.00 : ffff800010120444: ldr x25, [x25, #120] : */ : if (task_sleep && se && !throttled_hierarchy(cfs_rq)) : set_next_buddy(se); : break; : } : flags |= DEQUEUE_SLEEP; 0.00 : ffff800010120448: orr w20, w20, #0x1 : for_each_sched_entity(se) { 0.00 : ffff80001012044c: cbnz x25, ffff800010120098 0.00 : ffff800010120450: ldp x19, x20, [x29, #16] 0.00 : ffff800010120454: ldp x26, x27, [x29, #72] 0.00 : ffff800010120458: ldr w0, [x22, #4] 0.00 : ffff80001012045c: ldr x24, [x29, #56] 0.00 : ffff800010120460: ldr x28, [x29, #88] : : if (!se) : sub_nr_running(rq, 1); : : /* balance early to pull high priority tasks */ : if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 0.00 : ffff800010120464: ldr w1, [x29, #96] : sub_nr_running(): : sched_update_tick_dependency(rq); : } : : static inline void sub_nr_running(struct rq *rq, unsigned count) : { : rq->nr_running -= count; 0.00 : ffff800010120468: sub w0, w0, #0x1 0.00 : ffff80001012046c: str w0, [x22, #4] : dequeue_task_fair(): 0.00 : ffff800010120470: cbz w1, ffff8000101209ac : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120474: ldr x2, [x23, #440] : util_est_dequeue(): : ue.enqueued = cfs_rq->avg.util_est.enqueued; 0.00 : ffff800010120478: ldr w1, [x22, #312] : _task_util_est(): : return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 0.00 : ffff80001012047c: lsr x0, x2, #32 0.00 : ffff800010120480: cmp w0, w2 0.00 : ffff800010120484: csel w0, w0, w2, hi // hi = pmore 0.00 : ffff800010120488: orr w0, w0, #0x1 : util_est_dequeue(): : ue.enqueued -= min_t(unsigned int, ue.enqueued, _task_util_est(p)); 0.00 : ffff80001012048c: cmp w0, w1 0.00 : ffff800010120490: csel w0, w0, w1, ls // ls = plast 0.00 : ffff800010120494: sub w0, w1, w0 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010120498: str w0, [x22, #312] : util_est_dequeue(): : if (!task_sleep) 0.00 : ffff80001012049c: ldr w0, [x29, #100] 0.00 : ffff8000101204a0: cbz w0, ffff8000101204d4 : if (ue.enqueued & UTIL_AVG_UNCHANGED) 0.00 : ffff8000101204a4: ldr w0, [x23, #440] : ue = p->se.avg.util_est; 0.00 : ffff8000101204a8: ldr x2, [x23, #440] : if (ue.enqueued & UTIL_AVG_UNCHANGED) 0.00 : ffff8000101204ac: tbnz w0, #0, ffff8000101204d4 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101204b0: ldr x0, [x23, #432] : util_est_dequeue(): : ue = p->se.avg.util_est; 0.00 : ffff8000101204b4: ldr w1, [x23, #444] : ue.enqueued = (task_util(p) | UTIL_AVG_UNCHANGED); 0.00 : ffff8000101204b8: orr w0, w0, #0x1 : if (ue.ewma < ue.enqueued) { 0.00 : ffff8000101204bc: cmp w1, w0 0.00 : ffff8000101204c0: b.cs ffff80001012094c // b.hs, b.nlast 0.00 : ffff8000101204c4: mov w1, w0 : WRITE_ONCE(p->se.avg.util_est, ue); 0.00 : ffff8000101204c8: bfxil x2, x0, #0, #32 0.00 : ffff8000101204cc: bfi x2, x1, #32, #32 : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101204d0: str x2, [x23, #440] : dequeue_task_fair(): : rq->next_balance = jiffies; : : util_est_dequeue(&rq->cfs, p, task_sleep); : hrtick_update(rq); : } 0.00 : ffff8000101204d4: ldp x21, x22, [sp, #32] 0.00 : ffff8000101204d8: ldr x23, [sp, #48] 0.00 : ffff8000101204dc: ldr x25, [sp, #64] 0.00 : ffff8000101204e0: ldp x29, x30, [sp], #128 0.00 : ffff8000101204e4: ret : update_load_avg(): : } else if (decayed) { 0.00 : ffff8000101204e8: cbz w0, ffff800010120370 : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff8000101204ec: ldr x1, [x19, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff8000101204f0: add x0, x1, #0x80 0.00 : ffff8000101204f4: cmp x19, x0 0.00 : ffff8000101204f8: b.ne ffff800010120330 // b.any : cpufreq_update_util(): : */ : static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) : { : struct update_util_data *data; : : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff8000101204fc: ldrsw x4, [x1, #2568] 0.00 : ffff800010120500: adrp x2, ffff800011909000 0.00 : ffff800010120504: add x2, x2, #0x928 0.00 : ffff800010120508: adrp x0, ffff80001151f000 0.00 : ffff80001012050c: add x0, x0, #0x158 0.00 : ffff800010120510: ldr x2, [x2, x4, lsl #3] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120514: ldr x0, [x0, x2] : cpufreq_update_util(): : cpu_of(rq))); : if (data) 0.00 : ffff800010120518: cbz x0, ffff800010120330 : data->func(data, rq_clock(rq), flags); 0.00 : ffff80001012051c: ldr x4, [x0] 0.00 : ffff800010120520: mov w2, #0x0 // #0 0.00 : ffff800010120524: ldr x1, [x1, #2400] 0.00 : ffff800010120528: blr x4 0.00 : ffff80001012052c: b ffff800010120330 0.00 : ffff800010120530: str x1, [x29, #120] : rb_erase_cached(): : root->rb_leftmost = rb_next(node); 0.00 : ffff800010120534: mov x0, x28 0.00 : ffff800010120538: bl ffff800010cc5050 0.00 : ffff80001012053c: str x0, [x19, #64] 0.00 : ffff800010120540: ldr x1, [x29, #120] 0.00 : ffff800010120544: b ffff8000101203e4 : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff800010120548: add x0, x0, #0x140 0.00 : ffff80001012054c: b ffff800010128a04 : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff800010120550: ldr x0, [x19, #160] 0.00 : ffff800010120554: str x0, [x19, #256] 0.00 : ffff800010120558: b ffff800010120370 : update_tg_cfs_runnable(): : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff80001012055c: ldr x2, [x1] : u64 runnable_load_sum, load_sum = 0; 0.00 : ffff800010120560: mov x0, #0x0 // #0 : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010120564: lsr x2, x2, #10 0.00 : ffff800010120568: cbz x2, ffff800010120578 : div_s64_rem(): 0.00 : ffff80001012056c: ldr x1, [x1, #136] : *remainder = dividend % divisor; 0.00 : ffff800010120570: sxtw x0, w2 : return dividend / divisor; 0.00 : ffff800010120574: sdiv x0, x1, x0 : update_tg_cfs_runnable(): : runnable_sum = min(se->avg.load_sum, load_sum); 0.00 : ffff800010120578: ldr x2, [x25, #200] 0.00 : ffff80001012057c: cmp x2, x0 0.00 : ffff800010120580: csel x1, x2, x0, ls // ls = plast 0.00 : ffff800010120584: b ffff800010120224 : dequeue_task_fair(): : se = parent_entity(se); 0.00 : ffff800010120588: ldr x20, [x25, #120] : if (task_sleep && se && !throttled_hierarchy(cfs_rq)) 0.00 : ffff80001012058c: ldr w0, [x29, #100] 0.00 : ffff800010120590: cmp x20, #0x0 0.00 : ffff800010120594: csel w0, w0, wzr, ne // ne = any 0.00 : ffff800010120598: cbz w0, ffff800010120944 : set_next_buddy(): : } : } : : static void set_next_buddy(struct sched_entity *se) : { : if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 0.00 : ffff80001012059c: ldr x1, [x20, #136] 0.00 : ffff8000101205a0: mov x0, x20 0.00 : ffff8000101205a4: cbz x1, ffff800010120998 : return; : : for_each_sched_entity(se) { : if (SCHED_WARN_ON(!se->on_rq)) : return; : cfs_rq_of(se)->next = se; 0.00 : ffff8000101205a8: ldr x1, [x0, #128] 0.00 : ffff8000101205ac: str x0, [x1, #80] : for_each_sched_entity(se) { 0.00 : ffff8000101205b0: ldr x0, [x0, #120] 0.00 : ffff8000101205b4: cbnz x0, ffff8000101205a8 : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff8000101205b8: adrp x24, ffff800011afb000 0.00 : ffff8000101205bc: add x0, x24, #0xd80 : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff8000101205c0: adrp x27, ffff80001151f000 : update_tg_load_avg(): 0.00 : ffff8000101205c4: str x0, [x29, #112] : cpufreq_update_util(): 0.00 : ffff8000101205c8: add x0, x27, #0x158 0.00 : ffff8000101205cc: str x0, [x29, #104] : dequeue_task_fair(): : cfs_rq = cfs_rq_of(se); 0.00 : ffff8000101205d0: ldr x19, [x20, #128] : cfs_rq->idle_h_nr_running -= idle_h_nr_running; 0.00 : ffff8000101205d4: ldp w1, w0, [x19, #28] : cfs_rq_clock_pelt(): : return rq_clock_pelt(rq_of(cfs_rq)); 0.00 : ffff8000101205d8: ldr x2, [x19, #304] : dequeue_task_fair(): : cfs_rq->h_nr_running--; 0.00 : ffff8000101205dc: sub w1, w1, #0x1 : cfs_rq->idle_h_nr_running -= idle_h_nr_running; 0.00 : ffff8000101205e0: sub w0, w0, w21 0.00 : ffff8000101205e4: stp w1, w0, [x19, #28] : update_load_avg(): : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 0.00 : ffff8000101205e8: ldr x0, [x20, #192] : rq_clock_pelt(): : return rq->clock_pelt - rq->lost_idle_time; 0.00 : ffff8000101205ec: ldr x3, [x2, #2440] 0.00 : ffff8000101205f0: ldr x1, [x2, #2448] 0.00 : ffff8000101205f4: sub x27, x3, x1 : update_load_avg(): 0.00 : ffff8000101205f8: cbz x0, ffff80001012060c : __update_load_avg_se(now, cfs_rq, se); 0.00 : ffff8000101205fc: mov x2, x20 0.00 : ffff800010120600: mov x1, x19 0.00 : ffff800010120604: mov x0, x27 0.00 : ffff800010120608: bl ffff800010134868 <__update_load_avg_se> : update_cfs_rq_load_avg(): : if (cfs_rq->removed.nr) { 0.00 : ffff80001012060c: ldr w25, [x19, #196] 0.00 : ffff800010120610: cbz w25, ffff8000101206bc : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010120614: ldr w1, [x19, #156] 0.00 : ffff800010120618: mov w2, #0xb67e // #46718 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff80001012061c: add x25, x19, #0xc0 : u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 0.00 : ffff800010120620: add w26, w1, w2 : raw_spin_lock(&cfs_rq->removed.lock); 0.00 : ffff800010120624: mov x0, x25 0.00 : ffff800010120628: bl ffff800010cd80f0 <_raw_spin_lock> : cfs_rq->removed.nr = 0; 0.00 : ffff80001012062c: str wzr, [x19, #196] : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010120630: ldr x9, [x19, #216] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff800010120634: mov x0, x25 : swap(cfs_rq->removed.util_avg, removed_util); 0.00 : ffff800010120638: ldp x24, x28, [x19, #200] 0.00 : ffff80001012063c: stp xzr, xzr, [x19, #200] : swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 0.00 : ffff800010120640: str xzr, [x19, #216] 0.00 : ffff800010120644: str x9, [x29, #120] : raw_spin_unlock(&cfs_rq->removed.lock); 0.00 : ffff800010120648: bl ffff800010cd7c38 <_raw_spin_unlock> : __read_once_size(): 0.00 : ffff80001012064c: ldr x2, [x19, #160] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010120650: mov w0, w26 : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120654: ldr x5, [x19, #272] : cfs_rq->propagate = 1; 0.00 : ffff800010120658: mov x8, #0x1 // #1 : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff80001012065c: sub x4, x2, x24 : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120660: ldr x9, [x29, #120] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_avg, r); 0.00 : ffff800010120664: cmp x2, x4 : decayed = 1; 0.00 : ffff800010120668: mov w25, w8 : sub_positive(&sa->load_avg, r); 0.00 : ffff80001012066c: csel x4, x4, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120670: str x4, [x19, #160] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff800010120674: sub x4, x5, x9 0.00 : ffff800010120678: str x4, [x19, #272] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012067c: ldr x5, [x19, #136] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff800010120680: str x8, [x19, #264] : update_cfs_rq_load_avg(): : sub_positive(&sa->load_sum, r * divider); 0.00 : ffff800010120684: msub x0, x0, x24, x5 0.00 : ffff800010120688: cmp x5, x0 0.00 : ffff80001012068c: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120690: str x0, [x19, #136] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120694: ldr x4, [x19, #176] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_avg, r); 0.00 : ffff800010120698: sub x0, x4, x28 0.00 : ffff80001012069c: cmp x4, x0 0.00 : ffff8000101206a0: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101206a4: str x0, [x19, #176] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101206a8: ldr w4, [x19, #152] : update_cfs_rq_load_avg(): : sub_positive(&sa->util_sum, r * divider); 0.00 : ffff8000101206ac: msub w0, w26, w28, w4 0.00 : ffff8000101206b0: cmp w4, w0 0.00 : ffff8000101206b4: csel w0, w0, wzr, cs // cs = hs, nlast : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101206b8: str w0, [x19, #152] : update_cfs_rq_load_avg(): : decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 0.00 : ffff8000101206bc: mov x1, x19 0.00 : ffff8000101206c0: mov x0, x27 0.00 : ffff8000101206c4: bl ffff800010134a68 <__update_load_avg_cfs_rq> 0.00 : ffff8000101206c8: orr w0, w0, w25 : propagate_entity_load_avg(): : if (entity_is_task(se)) 0.00 : ffff8000101206cc: ldr x1, [x20, #136] 0.00 : ffff8000101206d0: cbz x1, ffff8000101208d4 : if (!gcfs_rq->propagate) 0.00 : ffff8000101206d4: ldr x2, [x1, #264] 0.00 : ffff8000101206d8: cbz x2, ffff8000101208d4 : gcfs_rq->propagate = 0; 0.00 : ffff8000101206dc: str xzr, [x1, #264] : add_tg_cfs_propagate(): : cfs_rq->propagate = 1; 0.00 : ffff8000101206e0: mov x4, #0x1 // #1 : propagate_entity_load_avg(): : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 0.00 : ffff8000101206e4: ldr x2, [x1, #272] : cfs_rq = cfs_rq_of(se); 0.00 : ffff8000101206e8: ldr x3, [x20, #128] : add_tg_cfs_propagate(): : cfs_rq->prop_runnable_sum += runnable_sum; 0.00 : ffff8000101206ec: ldr x0, [x3, #272] 0.00 : ffff8000101206f0: add x0, x0, x2 0.00 : ffff8000101206f4: stp x4, x0, [x3, #264] : propagate_entity_load_avg(): : update_tg_cfs_util(cfs_rq, se, gcfs_rq); 0.00 : ffff8000101206f8: ldr x2, [x1, #176] : update_tg_cfs_util(): : long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 0.00 : ffff8000101206fc: ldr x0, [x20, #240] 0.00 : ffff800010120700: sub x0, x2, x0 : if (!delta) 0.00 : ffff800010120704: cmp x0, #0x0 0.00 : ffff800010120708: b.eq ffff80001012073c // b.none : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff80001012070c: mov w4, #0xba7e // #47742 : se->avg.util_avg = gcfs_rq->avg.util_avg; 0.00 : ffff800010120710: str x2, [x20, #240] : se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010120714: mul w2, w2, w4 0.00 : ffff800010120718: str w2, [x20, #216] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff80001012071c: ldr x2, [x3, #176] : update_tg_cfs_util(): : add_positive(&cfs_rq->avg.util_avg, delta); 0.00 : ffff800010120720: add x0, x0, x2 0.00 : ffff800010120724: ccmp x2, x0, #0x2, lt // lt = tstop 0.00 : ffff800010120728: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff80001012072c: str x0, [x3, #176] : update_tg_cfs_util(): : cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 0.00 : ffff800010120730: ldr x0, [x3, #176] 0.00 : ffff800010120734: mul w4, w4, w0 0.00 : ffff800010120738: str w4, [x3, #152] : update_tg_cfs_runnable(): : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 0.00 : ffff80001012073c: ldr x0, [x1, #272] : if (!runnable_sum) 0.00 : ffff800010120740: cmp x0, #0x0 0.00 : ffff800010120744: b.eq ffff800010120860 // b.none : gcfs_rq->prop_runnable_sum = 0; 0.00 : ffff800010120748: str xzr, [x1, #272] : if (runnable_sum >= 0) { 0.00 : ffff80001012074c: b.lt ffff800010120918 // b.tstop : runnable_sum += se->avg.load_sum; 0.00 : ffff800010120750: ldr x2, [x20, #200] : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff800010120754: mov x1, #0xba7e // #47742 : runnable_sum += se->avg.load_sum; 0.00 : ffff800010120758: add x0, x0, x2 : runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 0.00 : ffff80001012075c: cmp x0, x1 0.00 : ffff800010120760: csel x1, x0, x1, le : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff800010120764: ldr w4, [x20, #216] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010120768: ldr x0, [x20] : update_tg_cfs_runnable(): : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff80001012076c: ldr x5, [x20, #224] : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 0.00 : ffff800010120770: lsr w4, w4, #10 : runnable_sum = max(runnable_sum, running_sum); 0.00 : ffff800010120774: cmp x4, x1 : se_weight(): 0.00 : ffff800010120778: lsr x0, x0, #10 : update_tg_cfs_runnable(): 0.00 : ffff80001012077c: csel x4, x4, x1, ge // ge = tcont : se->avg.load_sum = runnable_sum; 0.00 : ffff800010120780: str x4, [x20, #200] : load_sum = (s64)se_weight(se) * runnable_sum; 0.00 : ffff800010120784: mul x1, x0, x4 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 0.00 : ffff800010120788: msub x2, x2, x0, x1 : div_s64_rem(): 0.00 : ffff80001012078c: lsr x0, x1, #1 0.00 : ffff800010120790: mov x1, #0x5041 // #20545 0.00 : ffff800010120794: movk x1, #0xafb, lsl #16 0.00 : ffff800010120798: movk x1, #0x400, lsl #32 0.00 : ffff80001012079c: movk x1, #0xafb5, lsl #48 0.00 : ffff8000101207a0: umulh x0, x0, x1 0.00 : ffff8000101207a4: lsr x0, x0, #14 : update_tg_cfs_runnable(): : se->avg.load_avg = load_avg; 0.00 : ffff8000101207a8: str x0, [x20, #224] : delta_avg = load_avg - se->avg.load_avg; 0.00 : ffff8000101207ac: sub x0, x0, x5 : div_s64_rem(): 0.00 : ffff8000101207b0: mov x5, #0x5041 // #20545 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101207b4: ldr x1, [x3, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_avg, delta_avg); 0.00 : ffff8000101207b8: cmp x0, #0x0 : div_s64_rem(): 0.00 : ffff8000101207bc: movk x5, #0xafb, lsl #16 : update_tg_cfs_runnable(): 0.00 : ffff8000101207c0: add x0, x0, x1 : div_s64_rem(): 0.00 : ffff8000101207c4: movk x5, #0x400, lsl #32 : update_tg_cfs_runnable(): 0.00 : ffff8000101207c8: ccmp x1, x0, #0x2, lt // lt = tstop : div_s64_rem(): 0.00 : ffff8000101207cc: movk x5, #0xafb5, lsl #48 : update_tg_cfs_runnable(): 0.00 : ffff8000101207d0: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101207d4: str x0, [x3, #160] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.load_sum, delta_sum); 0.00 : ffff8000101207d8: cmp x2, #0x0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101207dc: ldr x1, [x3, #136] : update_tg_cfs_runnable(): 0.00 : ffff8000101207e0: add x0, x2, x1 0.00 : ffff8000101207e4: ccmp x1, x0, #0x2, lt // lt = tstop 0.00 : ffff8000101207e8: csel x0, x0, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000101207ec: str x0, [x3, #136] : se_runnable(): : return scale_load_down(se->runnable_weight); 0.00 : ffff8000101207f0: ldr x1, [x20, #16] : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff8000101207f4: ldr w2, [x20, #64] : se_runnable(): 0.00 : ffff8000101207f8: lsr x1, x1, #10 : update_tg_cfs_runnable(): : runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 0.00 : ffff8000101207fc: mul x1, x1, x4 : div_s64_rem(): 0.00 : ffff800010120800: lsr x0, x1, #1 0.00 : ffff800010120804: umulh x0, x0, x5 0.00 : ffff800010120808: lsr x0, x0, #14 : update_tg_cfs_runnable(): : if (se->on_rq) { 0.00 : ffff80001012080c: cbz w2, ffff800010120858 : delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 0.00 : ffff800010120810: ldr x6, [x20, #232] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120814: ldr x7, [x3, #168] : se_weight(): : return scale_load_down(se->load.weight); 0.00 : ffff800010120818: ldr x2, [x20] : update_tg_cfs_runnable(): 0.00 : ffff80001012081c: sub x6, x0, x6 : se_weight(se) * se->avg.runnable_load_sum; 0.00 : ffff800010120820: ldr x8, [x20, #208] : add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 0.00 : ffff800010120824: add x5, x6, x7 0.00 : ffff800010120828: cmp x6, #0x0 0.00 : ffff80001012082c: ccmp x7, x5, #0x2, lt // lt = tstop : se_weight(): 0.00 : ffff800010120830: lsr x2, x2, #10 : update_tg_cfs_runnable(): 0.00 : ffff800010120834: csel x5, x5, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120838: str x5, [x3, #168] : update_tg_cfs_runnable(): : delta_sum = runnable_load_sum - 0.00 : ffff80001012083c: msub x1, x2, x8, x1 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff800010120840: ldr x2, [x3, #144] : update_tg_cfs_runnable(): : add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 0.00 : ffff800010120844: cmp x1, #0x0 0.00 : ffff800010120848: add x1, x1, x2 0.00 : ffff80001012084c: ccmp x2, x1, #0x2, lt // lt = tstop 0.00 : ffff800010120850: csel x1, x1, xzr, cs // cs = hs, nlast : __write_once_size(): : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff800010120854: str x1, [x3, #144] : update_tg_cfs_runnable(): : se->avg.runnable_load_sum = runnable_sum; 0.00 : ffff800010120858: str x4, [x20, #208] : se->avg.runnable_load_avg = runnable_load_avg; 0.00 : ffff80001012085c: str x0, [x20, #232] : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff800010120860: ldr x1, [x19, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff800010120864: add x0, x1, #0x80 0.00 : ffff800010120868: cmp x19, x0 0.00 : ffff80001012086c: b.eq ffff8000101208e8 // b.none : update_tg_load_avg(): : if (cfs_rq->tg == &root_task_group) 0.00 : ffff800010120870: ldr x0, [x19, #336] 0.00 : ffff800010120874: ldr x1, [x29, #112] 0.00 : ffff800010120878: cmp x0, x1 0.00 : ffff80001012087c: b.eq ffff8000101208c0 // b.none : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 0.00 : ffff800010120880: ldr x1, [x19, #160] 0.00 : ffff800010120884: ldr x2, [x19, #256] : if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 0.00 : ffff800010120888: subs x1, x1, x2 0.00 : ffff80001012088c: cneg x3, x1, mi // mi = first 0.00 : ffff800010120890: cmp x3, x2, lsr #6 0.00 : ffff800010120894: b.ls ffff8000101208c0 // b.plast : atomic_long_add(delta, &cfs_rq->tg->load_avg); 0.00 : ffff800010120898: add x2, x0, #0x140 : arch_static_branch_jump(): 0.00 : ffff80001012089c: b ffff8000101208ac 0.00 : ffff8000101208a0: b ffff8000101208ac : __lse_atomic64_add(): 0.00 : ffff8000101208a4: stadd x1, [x2] 0.00 : ffff8000101208a8: b ffff8000101208b4 : __ll_sc_atomic64_add(): 0.00 : ffff8000101208ac: add x0, x0, #0x140 0.00 : ffff8000101208b0: b ffff800010128a1c : update_tg_load_avg(): : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 0.00 : ffff8000101208b4: ldr x0, [x19, #160] 0.00 : ffff8000101208b8: str x0, [x19, #256] 0.00 : ffff8000101208bc: nop : dequeue_task_fair(): : update_cfs_group(se); 0.00 : ffff8000101208c0: mov x0, x20 0.00 : ffff8000101208c4: bl ffff80001011fd78 : for_each_sched_entity(se) { 0.00 : ffff8000101208c8: ldr x20, [x20, #120] 0.00 : ffff8000101208cc: cbnz x20, ffff8000101205d0 0.00 : ffff8000101208d0: b ffff800010120450 : update_load_avg(): : } else if (decayed) { 0.00 : ffff8000101208d4: cbz w0, ffff8000101208c0 : cfs_rq_util_change(): : struct rq *rq = rq_of(cfs_rq); 0.00 : ffff8000101208d8: ldr x1, [x19, #304] : if (&rq->cfs == cfs_rq) { 0.00 : ffff8000101208dc: add x0, x1, #0x80 0.00 : ffff8000101208e0: cmp x19, x0 0.00 : ffff8000101208e4: b.ne ffff800010120870 // b.any : cpufreq_update_util(): : data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 0.00 : ffff8000101208e8: ldrsw x3, [x1, #2568] 0.00 : ffff8000101208ec: adrp x2, ffff800011909000 0.00 : ffff8000101208f0: add x2, x2, #0x928 0.00 : ffff8000101208f4: ldr x0, [x29, #104] 0.00 : ffff8000101208f8: ldr x2, [x2, x3, lsl #3] : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101208fc: ldr x0, [x0, x2] : cpufreq_update_util(): : if (data) 0.00 : ffff800010120900: cbz x0, ffff800010120870 : data->func(data, rq_clock(rq), flags); 0.00 : ffff800010120904: ldr x3, [x0] 0.00 : ffff800010120908: mov w2, #0x0 // #0 0.00 : ffff80001012090c: ldr x1, [x1, #2400] 0.00 : ffff800010120910: blr x3 0.00 : ffff800010120914: b ffff800010120870 : update_tg_cfs_runnable(): : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010120918: ldr x2, [x1] : u64 runnable_load_sum, load_sum = 0; 0.00 : ffff80001012091c: mov x0, #0x0 // #0 : if (scale_load_down(gcfs_rq->load.weight)) { 0.00 : ffff800010120920: lsr x2, x2, #10 0.00 : ffff800010120924: cbz x2, ffff800010120934 : div_s64_rem(): 0.00 : ffff800010120928: ldr x0, [x1, #136] : *remainder = dividend % divisor; 0.00 : ffff80001012092c: sxtw x1, w2 : return dividend / divisor; 0.00 : ffff800010120930: sdiv x0, x0, x1 : update_tg_cfs_runnable(): : runnable_sum = min(se->avg.load_sum, load_sum); 0.00 : ffff800010120934: ldr x2, [x20, #200] 0.00 : ffff800010120938: cmp x2, x0 0.00 : ffff80001012093c: csel x1, x2, x0, ls // ls = plast 0.00 : ffff800010120940: b ffff800010120764 : dequeue_task_fair(): : for_each_sched_entity(se) { 0.00 : ffff800010120944: cbnz x20, ffff8000101205b8 0.00 : ffff800010120948: b ffff800010120450 : util_est_dequeue(): : last_ewma_diff = ue.enqueued - ue.ewma; 47.97 : ffff80001012094c: sub w4, w0, w1 : within_margin(): : return ((unsigned int)(value + margin - 1) < (2 * margin - 1)); 0.00 : ffff800010120950: add w3, w4, #0x9 : util_est_dequeue(): : if (within_margin(last_ewma_diff, (SCHED_CAPACITY_SCALE / 100))) 0.00 : ffff800010120954: cmp w3, #0x12 0.00 : ffff800010120958: b.ls ffff8000101204d4 // b.plast : cpu = cpu_of(rq_of(cfs_rq)); 0.00 : ffff80001012095c: ldr x7, [x22, #432] : capacity_orig_of(): : #endif : : #ifdef CONFIG_SMP : static inline unsigned long capacity_orig_of(int cpu) : { : return cpu_rq(cpu)->cpu_capacity_orig; 0.00 : ffff800010120960: adrp x5, ffff800011909000 0.00 : ffff800010120964: add x5, x5, #0x928 0.00 : ffff800010120968: adrp x3, ffff800011528000 0.00 : ffff80001012096c: add x3, x3, #0xe80 : __read_once_size(): 0.00 : ffff800010120970: ldr x6, [x23, #432] : capacity_orig_of(): 0.00 : ffff800010120974: ldrsw x7, [x7, #2568] 0.00 : ffff800010120978: ldr x5, [x5, x7, lsl #3] 0.00 : ffff80001012097c: add x3, x5, x3 : util_est_dequeue(): : if (task_util(p) > capacity_orig_of(cpu)) 0.00 : ffff800010120980: ldr x3, [x3, #2488] 0.00 : ffff800010120984: cmp x6, x3 0.00 : ffff800010120988: b.hi ffff8000101204d4 // b.pmore : ue.ewma += last_ewma_diff; 0.00 : ffff80001012098c: add w1, w4, w1, lsl #2 : ue.ewma >>= UTIL_EST_WEIGHT_SHIFT; 0.00 : ffff800010120990: lsr w1, w1, #2 0.00 : ffff800010120994: b ffff8000101204c8 : set_next_buddy(): : if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 0.00 : ffff800010120998: ldr w0, [x20, #536] 0.00 : ffff80001012099c: cmp w0, #0x5 0.00 : ffff8000101209a0: b.eq ffff8000101205b8 // b.none 0.00 : ffff8000101209a4: mov x0, x20 0.00 : ffff8000101209a8: b ffff8000101205a8 : sched_idle_rq(): : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 0.00 : ffff8000101209ac: ldr w1, [x22, #160] 0.00 : ffff8000101209b0: cmp w0, #0x0 0.00 : ffff8000101209b4: ccmp w1, w0, #0x0, ne // ne = any 0.00 : ffff8000101209b8: b.ne ffff800010120474 // b.any : dequeue_task_fair(): : rq->next_balance = jiffies; 0.00 : ffff8000101209bc: adrp x0, ffff800011907000 0.00 : ffff8000101209c0: ldr x0, [x0, #2432] 0.00 : ffff8000101209c4: str x0, [x22, #2376] 0.00 : ffff8000101209c8: b ffff800010120474 Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011bf38 : : arch_cpu_idle_exit(): : : return 1; : } : : /* Weak implementations for optional arch specific functions */ : void __weak arch_cpu_idle_prepare(void) { } 100.00 : ffff80001011bf38: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010197170 : : cgroup_base_stat_cputime_account_end(): : rstatc = get_cpu_ptr(cgrp->rstat_cpu); : u64_stats_update_begin(&rstatc->bsync); : return rstatc; : } : : static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp, 0.00 : ffff800010197170: stp x29, x30, [sp, #-16]! : struct cgroup_rstat_cpu *rstatc) : { : u64_stats_update_end(&rstatc->bsync); : cgroup_rstat_updated(cgrp, smp_processor_id()); 0.00 : ffff800010197174: adrp x1, ffff80001151d000 0.00 : ffff800010197178: add x1, x1, #0x18 : static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp, 0.00 : ffff80001019717c: mov x29, sp : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff800010197180: mrs x2, tpidr_el1 : cgroup_base_stat_cputime_account_end(): : cgroup_rstat_updated(cgrp, smp_processor_id()); 0.00 : ffff800010197184: ldr w1, [x1, x2] 0.00 : ffff800010197188: bl ffff800010196d80 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff80001019718c: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010197190: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff800010197194: sub x0, x0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff800010197198: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff80001019719c: cbz x0, ffff8000101971a8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101971a0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 100.00 : ffff8000101971a4: cbnz x0, ffff8000101971ac : cgroup_base_stat_cputime_account_end(): : put_cpu_ptr(rstatc); 0.00 : ffff8000101971a8: bl ffff800010cd2e50 : } 0.00 : ffff8000101971ac: ldp x29, x30, [sp], #16 0.00 : ffff8000101971b0: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010712a00 : : iommu_map_atomic(): : } : EXPORT_SYMBOL_GPL(iommu_map); : : int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, : phys_addr_t paddr, size_t size, int prot) : { 100.00 : ffff800010712a00: stp x29, x30, [sp, #-16]! : return __iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC); 0.00 : ffff800010712a04: mov w5, #0xa20 // #2592 : { 0.00 : ffff800010712a08: mov x29, sp : return __iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC); 0.00 : ffff800010712a0c: bl ffff800010712828 <__iommu_map> : } 0.00 : ffff800010712a10: ldp x29, x30, [sp], #16 0.00 : ffff800010712a14: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff80001011ba88 : : account_idle_ticks(): : /* : * Account multiple ticks of idle time. : * @ticks: number of stolen ticks : */ : void account_idle_ticks(unsigned long ticks) : { 0.00 : ffff80001011ba88: stp x29, x30, [sp, #-32]! : u64 cputime, steal; : : if (sched_clock_irqtime) { 0.00 : ffff80001011ba8c: adrp x1, ffff800011afb000 : { 0.00 : ffff80001011ba90: mov x29, sp : if (sched_clock_irqtime) { 0.00 : ffff80001011ba94: ldr w1, [x1, #3992] 0.00 : ffff80001011ba98: cbnz w1, ffff80001011bb44 0.00 : ffff80001011ba9c: str x19, [x29, #16] : irqtime_account_idle_ticks(ticks); : return; : } : : cputime = ticks * TICK_NSEC; 0.00 : ffff80001011baa0: mov x19, #0x900 // #2304 0.00 : ffff80001011baa4: movk x19, #0x3d, lsl #16 0.00 : ffff80001011baa8: mul x19, x0, x19 : arch_static_branch(): : #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE : : static __always_inline bool arch_static_branch(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff80001011baac: nop : steal_account_process_time(): : return 0; 0.00 : ffff80001011bab0: mov x0, #0x0 // #0 0.00 : ffff80001011bab4: b ffff80001011bb1c : paravirt_steal_clock(): : : extern struct paravirt_patch_template pv_ops; : : static inline u64 paravirt_steal_clock(int cpu) : { : return pv_ops.time.steal_clock(cpu); 0.00 : ffff80001011bab8: adrp x0, ffff800011af8000 : steal_account_process_time(): : steal = paravirt_steal_clock(smp_processor_id()); 0.00 : ffff80001011babc: adrp x1, ffff80001151d000 0.00 : ffff80001011bac0: add x1, x1, #0x18 : paravirt_steal_clock(): 0.00 : ffff80001011bac4: ldr x2, [x0, #248] : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011bac8: mrs x0, tpidr_el1 : paravirt_steal_clock(): 0.00 : ffff80001011bacc: ldr w0, [x1, x0] 0.00 : ffff80001011bad0: blr x2 : __my_cpu_offset(): : "mrs %0, tpidr_el2", : ARM64_HAS_VIRT_HOST_EXTN) : : "=r" (off) : : "Q" (*(const unsigned long *)current_stack_pointer)); 0.00 : ffff80001011bad4: mov x5, sp : steal_account_process_time(): : steal -= this_rq()->prev_steal_time; 0.00 : ffff80001011bad8: adrp x1, ffff800011528000 0.00 : ffff80001011badc: add x1, x1, #0xe80 : __my_cpu_offset(): : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff80001011bae0: mrs x2, tpidr_el1 : steal_account_process_time(): 0.00 : ffff80001011bae4: add x4, x1, x2 : account_steal_time(): : u64 *cpustat = kcpustat_this_cpu->cpustat; 0.00 : ffff80001011bae8: adrp x3, ffff80001151e000 0.00 : ffff80001011baec: add x3, x3, #0xfc0 0.00 : ffff80001011baf0: add x3, x3, x2 : steal_account_process_time(): : steal -= this_rq()->prev_steal_time; 0.00 : ffff80001011baf4: ldr x4, [x4, #2848] 0.00 : ffff80001011baf8: sub x0, x0, x4 : account_steal_time(): : cpustat[CPUTIME_STEAL] += cputime; 0.00 : ffff80001011bafc: ldr x2, [x3, #56] 0.00 : ffff80001011bb00: add x2, x2, x0 0.00 : ffff80001011bb04: str x2, [x3, #56] : __my_cpu_offset(): 0.00 : ffff80001011bb08: mrs x2, tpidr_el1 : steal_account_process_time(): : this_rq()->prev_steal_time += steal; 0.00 : ffff80001011bb0c: add x1, x1, x2 0.00 : ffff80001011bb10: ldr x2, [x1, #2848] 0.00 : ffff80001011bb14: add x2, x2, x0 0.00 : ffff80001011bb18: str x2, [x1, #2848] : account_idle_ticks(): : steal = steal_account_process_time(ULONG_MAX); : : if (steal >= cputime) 0.00 : ffff80001011bb1c: cmp x19, x0 0.00 : ffff80001011bb20: b.hi ffff80001011bb30 // b.pmore 0.00 : ffff80001011bb24: ldr x19, [x29, #16] : return; : : cputime -= steal; : account_idle_time(cputime); : } 0.00 : ffff80001011bb28: ldp x29, x30, [sp], #32 0.00 : ffff80001011bb2c: ret : account_idle_time(cputime); 0.00 : ffff80001011bb30: sub x0, x19, x0 0.00 : ffff80001011bb34: bl ffff80001011b620 0.00 : ffff80001011bb38: ldr x19, [x29, #16] : } 0.00 : ffff80001011bb3c: ldp x29, x30, [sp], #32 0.00 : ffff80001011bb40: ret : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 100.00 : ffff80001011bb44: mrs x3, sp_el0 : irqtime_account_idle_ticks(): : irqtime_account_process_tick(current, 0, ticks); 0.00 : ffff80001011bb48: mov w2, w0 0.00 : ffff80001011bb4c: mov w1, #0x0 // #0 0.00 : ffff80001011bb50: mov x0, x3 0.00 : ffff80001011bb54: bl ffff80001011b668 : account_idle_ticks(): : } 0.00 : ffff80001011bb58: ldp x29, x30, [sp], #32 0.00 : ffff80001011bb5c: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000100a3868 <__set_fixmap>: : __set_fixmap(): : * ever need to use IPIs for TLB broadcasting, then we're in trouble here. : */ : void __set_fixmap(enum fixed_addresses idx, : phys_addr_t phys, pgprot_t flags) : { : unsigned long addr = __fix_to_virt(idx); 0.00 : ffff8000100a3868: mov x4, #0xfea00000 // #4271898624 : pte_t *ptep; : : BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 0.00 : ffff8000100a386c: sub w3, w0, #0x1 : unsigned long addr = __fix_to_virt(idx); 0.00 : ffff8000100a3870: movk x4, #0xfdff, lsl #32 0.00 : ffff8000100a3874: lsl w0, w0, #12 0.00 : ffff8000100a3878: movk x4, #0xffff, lsl #48 : BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 0.00 : ffff8000100a387c: cmp w3, #0x5c9 : unsigned long addr = __fix_to_virt(idx); 0.00 : ffff8000100a3880: sub x4, x4, x0 : BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 0.00 : ffff8000100a3884: b.hi ffff8000100a3924 <__set_fixmap+0xbc> // b.pmore : fixmap_pte(): : return &bm_pte[pte_index(addr)]; 0.00 : ffff8000100a3888: lsr x3, x4, #12 0.00 : ffff8000100a388c: adrp x5, ffff800011af0000 0.00 : ffff8000100a3890: and x6, x3, #0x1ff 0.00 : ffff8000100a3894: add x5, x5, #0x0 : __set_fixmap(): : : ptep = fixmap_pte(addr); : : if (pgprot_val(flags)) { 0.00 : ffff8000100a3898: cbz x2, ffff8000100a38c8 <__set_fixmap+0x60> : set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); 0.00 : ffff8000100a389c: and x1, x1, #0xfffffffffffff000 : set_pte(): : : /* : * Only if the new pte is valid and kernel, otherwise TLB maintenance : * or update_mmu_cache() have the necessary barriers. : */ : if (pte_valid_not_user(pte)) { 0.00 : ffff8000100a38a0: mov x0, #0x41 // #65 : __set_fixmap(): 0.00 : ffff8000100a38a4: orr x2, x1, x2 : __write_once_size(): : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; : case 8: *(volatile __u64 *)p = *(__u64 *)res; break; 0.00 : ffff8000100a38a8: str x2, [x5, x6, lsl #3] : set_pte(): 0.00 : ffff8000100a38ac: and x2, x2, x0 0.00 : ffff8000100a38b0: cmp x2, #0x1 0.00 : ffff8000100a38b4: b.eq ffff8000100a38bc <__set_fixmap+0x54> // b.none : __set_fixmap(): : } else { : pte_clear(&init_mm, addr, ptep); : flush_tlb_kernel_range(addr, addr+PAGE_SIZE); : } : } 0.00 : ffff8000100a38b8: ret : set_pte(): : dsb(ishst); 0.00 : ffff8000100a38bc: dsb ishst : isb(); 0.00 : ffff8000100a38c0: isb : __set_fixmap(): 0.00 : ffff8000100a38c4: ret : flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 0.00 : ffff8000100a38c8: mov x1, #0xffffffffffff1000 // #-61440 : __write_once_size(): 0.00 : ffff8000100a38cc: str xzr, [x5, x6, lsl #3] : __set_fixmap(): 0.00 : ffff8000100a38d0: movk x1, #0xfea0, lsl #16 0.00 : ffff8000100a38d4: movk x1, #0xfdff, lsl #32 0.00 : ffff8000100a38d8: sub x1, x1, x0 : flush_tlb_kernel_range(): : : static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) : { : unsigned long addr; : : if ((end - start) > (MAX_TLBI_OPS * PAGE_SIZE)) { 0.00 : ffff8000100a38dc: sub x4, x1, x4 0.00 : ffff8000100a38e0: cmp x4, #0x200, lsl #12 0.00 : ffff8000100a38e4: b.hi ffff8000100a3928 <__set_fixmap+0xc0> // b.pmore : flush_tlb_all(); : return; : } : : start = __TLBI_VADDR(start, 0); 0.00 : ffff8000100a38e8: and x0, x3, #0xfffffffffff : end = __TLBI_VADDR(end, 0); 0.00 : ffff8000100a38ec: ubfx x1, x1, #12, #44 : : dsb(ishst); 0.00 : ffff8000100a38f0: dsb ishst : for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) 0.00 : ffff8000100a38f4: cmp x0, x1 0.00 : ffff8000100a38f8: b.cs ffff8000100a3918 <__set_fixmap+0xb0> // b.hs, b.nlast 0.00 : ffff8000100a38fc: nop : __tlbi(vaale1is, addr); 0.00 : ffff8000100a3900: tlbi vaale1is, x0 0.00 : ffff8000100a3904: nop 0.00 : ffff8000100a3908: nop : for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) 0.00 : ffff8000100a390c: add x0, x0, #0x1 0.00 : ffff8000100a3910: cmp x1, x0 0.00 : ffff8000100a3914: b.ne ffff8000100a3900 <__set_fixmap+0x98> // b.any : dsb(ish); 0.00 : ffff8000100a3918: dsb ish : isb(); 100.00 : ffff8000100a391c: isb : __set_fixmap(): : } 0.00 : ffff8000100a3920: ret : BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 0.00 : ffff8000100a3924: brk #0x800 : flush_tlb_all(): : dsb(ishst); 0.00 : ffff8000100a3928: dsb ishst : __tlbi(vmalle1is); 0.00 : ffff8000100a392c: tlbi vmalle1is 0.00 : ffff8000100a3930: nop 0.00 : ffff8000100a3934: nop : dsb(ish); 0.00 : ffff8000100a3938: dsb ish : isb(); 0.00 : ffff8000100a393c: isb : __set_fixmap(): : } 0.00 : ffff8000100a3940: ret Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010cc8498 : : strscpy(): : { : const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; : size_t max = count; : long res = 0; : : if (count == 0 || WARN_ON_ONCE(count > INT_MAX)) 0.00 : ffff800010cc8498: cbz x2, ffff800010cc85a0 0.00 : ffff800010cc849c: mov x3, #0x7fffffff // #2147483647 0.00 : ffff800010cc84a0: cmp x2, x3 0.00 : ffff800010cc84a4: b.hi ffff800010cc859c // b.pmore : #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS : /* : * If src is unaligned, don't cross a page boundary, : * since we don't know if the next page is mapped. : */ : if ((long)src & (sizeof(long) - 1)) { 0.00 : ffff800010cc84a8: tst x1, #0x7 0.00 : ffff800010cc84ac: b.ne ffff800010cc8570 // b.any : /* If src or dest is unaligned, don't do word-at-a-time. */ : if (((long) dest | (long) src) & (sizeof(long) - 1)) : max = 0; : #endif : : while (max >= sizeof(unsigned long)) { 0.00 : ffff800010cc84b0: cmp x2, #0x7 0.00 : ffff800010cc84b4: b.ls ffff800010cc858c // b.plast 0.00 : ffff800010cc84b8: mov x4, x2 : read_word_at_a_time(): : : static __no_kasan_or_inline : unsigned long read_word_at_a_time(const void *addr) : { : kasan_check_read(addr, 1); : return *(unsigned long *)addr; 0.00 : ffff800010cc84bc: ldr x5, [x1] : has_zero(): : #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) } : : static inline unsigned long has_zero(unsigned long a, unsigned long *bits, : const struct word_at_a_time *c) : { : unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits; 0.00 : ffff800010cc84c0: mov x7, #0xfefefefefefefefe // #-72340172838076674 0.00 : ffff800010cc84c4: movk x7, #0xfeff 0.00 : ffff800010cc84c8: add x3, x5, x7 0.00 : ffff800010cc84cc: bic x3, x3, x5 : strscpy(): : unsigned long c, data; : : c = read_word_at_a_time(src+res); : if (has_zero(c, &data, &constants)) { 0.00 : ffff800010cc84d0: ands x3, x3, #0x8080808080808080 0.00 : ffff800010cc84d4: b.ne ffff800010cc85e8 // b.any 0.00 : ffff800010cc84d8: sub x6, x4, #0x8 0.00 : ffff800010cc84dc: sub x2, x2, #0x8 0.00 : ffff800010cc84e0: and x6, x6, #0xfffffffffffffff8 : c = read_word_at_a_time(src+res); 0.00 : ffff800010cc84e4: mov x4, #0x0 // #0 0.00 : ffff800010cc84e8: sub x6, x2, x6 0.00 : ffff800010cc84ec: b ffff800010cc8508 : read_word_at_a_time(): 0.00 : ffff800010cc84f0: ldr x5, [x1, x4] 0.00 : ffff800010cc84f4: sub x2, x2, #0x8 : has_zero(): 0.00 : ffff800010cc84f8: add x3, x5, x7 0.00 : ffff800010cc84fc: bic x3, x3, x5 : strscpy(): : if (has_zero(c, &data, &constants)) { 0.00 : ffff800010cc8500: ands x3, x3, #0x8080808080808080 0.00 : ffff800010cc8504: b.ne ffff800010cc85ac // b.any : data = prep_zero_mask(c, data, &constants); : data = create_zero_mask(data); : *(unsigned long *)(dest+res) = c & zero_bytemask(data); : return res + find_zero(data); : } : *(unsigned long *)(dest+res) = c; 100.00 : ffff800010cc8508: str x5, [x0, x4] : while (max >= sizeof(unsigned long)) { 0.00 : ffff800010cc850c: cmp x2, x6 : res += sizeof(unsigned long); 0.00 : ffff800010cc8510: add x4, x4, #0x8 : while (max >= sizeof(unsigned long)) { 0.00 : ffff800010cc8514: b.ne ffff800010cc84f0 // b.any : count -= sizeof(unsigned long); : max -= sizeof(unsigned long); : } : : while (count) { 0.00 : ffff800010cc8518: cbz x2, ffff800010cc8558 0.00 : ffff800010cc851c: add x5, x0, x4 0.00 : ffff800010cc8520: add x3, x1, x4 : char c; : : c = src[res]; 0.00 : ffff800010cc8524: ldrb w3, [x3] : dest[res] = c; 0.00 : ffff800010cc8528: strb w3, [x5] : if (!c) 0.00 : ffff800010cc852c: cbz w3, ffff800010cc85a4 0.00 : ffff800010cc8530: add x3, x4, #0x1 0.00 : ffff800010cc8534: add x2, x4, x2 0.00 : ffff800010cc8538: b ffff800010cc854c : c = src[res]; 0.00 : ffff800010cc853c: ldrb w5, [x1, x3] : dest[res] = c; 0.00 : ffff800010cc8540: strb w5, [x0, x3] 0.00 : ffff800010cc8544: add x3, x3, #0x1 : if (!c) 0.00 : ffff800010cc8548: cbz w5, ffff800010cc85a4 : return res; : res++; 0.00 : ffff800010cc854c: mov x4, x3 : while (count) { 0.00 : ffff800010cc8550: cmp x2, x3 0.00 : ffff800010cc8554: b.ne ffff800010cc853c // b.any : count--; : } : : /* Hit buffer length without finding a NUL; force NUL-termination. */ : if (res) 0.00 : ffff800010cc8558: cbz x4, ffff800010cc85a0 : dest[res-1] = '\0'; 0.00 : ffff800010cc855c: add x0, x0, x4 : : return -E2BIG; 0.00 : ffff800010cc8560: mov x4, #0xfffffffffffffff9 // #-7 : dest[res-1] = '\0'; 0.00 : ffff800010cc8564: sturb wzr, [x0, #-1] : } 0.00 : ffff800010cc8568: mov x0, x4 0.00 : ffff800010cc856c: ret : size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)); 0.00 : ffff800010cc8570: and x3, x1, #0xfff 0.00 : ffff800010cc8574: mov x4, #0x1000 // #4096 0.00 : ffff800010cc8578: sub x4, x4, x3 0.00 : ffff800010cc857c: cmp x4, x2 0.00 : ffff800010cc8580: csel x4, x4, x2, ls // ls = plast : while (max >= sizeof(unsigned long)) { 0.00 : ffff800010cc8584: cmp x4, #0x7 0.00 : ffff800010cc8588: b.hi ffff800010cc84bc // b.pmore 0.00 : ffff800010cc858c: mov x3, x1 0.00 : ffff800010cc8590: mov x5, x0 0.00 : ffff800010cc8594: mov x4, #0x0 // #0 0.00 : ffff800010cc8598: b ffff800010cc8524 : if (count == 0 || WARN_ON_ONCE(count > INT_MAX)) 0.00 : ffff800010cc859c: brk #0x800 : return -E2BIG; 0.00 : ffff800010cc85a0: mov x4, #0xfffffffffffffff9 // #-7 : } 0.00 : ffff800010cc85a4: mov x0, x4 0.00 : ffff800010cc85a8: ret 0.00 : ffff800010cc85ac: add x0, x0, x4 : create_zero_mask(): : : #define prep_zero_mask(a, bits, c) (bits) : : static inline unsigned long create_zero_mask(unsigned long bits) : { : bits = (bits - 1) & ~bits; 0.00 : ffff800010cc85b0: sub x1, x3, #0x1 0.00 : ffff800010cc85b4: bic x3, x1, x3 : return bits >> 7; 0.00 : ffff800010cc85b8: lsr x3, x3, #7 : strscpy(): : *(unsigned long *)(dest+res) = c & zero_bytemask(data); 0.00 : ffff800010cc85bc: and x5, x3, x5 0.00 : ffff800010cc85c0: str x5, [x0] : fls64(): : return fls(x); : } : #elif BITS_PER_LONG == 64 : static __always_inline int fls64(__u64 x) : { : if (x == 0) 0.00 : ffff800010cc85c4: cbz x3, ffff800010cc85a4 : __fls(): : * : * Undefined if no set bit exists, so code should check against 0 first. : */ : static __always_inline unsigned long __fls(unsigned long word) : { : return (sizeof(word) * 8) - 1 - __builtin_clzl(word); 0.00 : ffff800010cc85c8: clz x3, x3 0.00 : ffff800010cc85cc: mov x0, #0x3f // #63 0.00 : ffff800010cc85d0: sub x3, x0, x3 : fls64(): : return 0; : return __fls(x) + 1; 0.00 : ffff800010cc85d4: add w0, w3, #0x1 0.00 : ffff800010cc85d8: asr w0, w0, #3 0.00 : ffff800010cc85dc: add x4, x4, w0, sxtw : strscpy(): : } 0.00 : ffff800010cc85e0: mov x0, x4 0.00 : ffff800010cc85e4: ret : c = read_word_at_a_time(src+res); 0.00 : ffff800010cc85e8: mov x4, #0x0 // #0 0.00 : ffff800010cc85ec: b ffff800010cc85b0 Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101f5088 : : refresh_cpu_vm_stats(): : * bouncing and will have to be only done when necessary. : * : * The function returns the number of global counters updated. : */ : static int refresh_cpu_vm_stats(bool do_pagesets) : { 0.00 : ffff8000101f5088: stp x29, x30, [sp, #-352]! 0.00 : ffff8000101f508c: and w0, w0, #0xff 0.00 : ffff8000101f5090: adrp x1, ffff800011909000 0.00 : ffff8000101f5094: add x1, x1, #0x908 0.00 : ffff8000101f5098: mov x29, sp 0.00 : ffff8000101f509c: stp x19, x20, [sp, #16] 0.00 : ffff8000101f50a0: stp x21, x22, [sp, #32] 0.00 : ffff8000101f50a4: stp x23, x24, [sp, #48] 0.00 : ffff8000101f50a8: stp x25, x26, [sp, #64] 0.00 : ffff8000101f50ac: stp x27, x28, [sp, #80] 0.00 : ffff8000101f50b0: str w0, [x29, #116] 0.00 : ffff8000101f50b4: ldr x0, [x1] 0.00 : ffff8000101f50b8: str x0, [x29, #344] 0.00 : ffff8000101f50bc: mov x0, #0x0 // #0 : struct pglist_data *pgdat; : struct zone *zone; : int i; : int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; : #ifdef CONFIG_NUMA : int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, }; 0.00 : ffff8000101f50c0: stp xzr, xzr, [x29, #144] : #endif : int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 0.00 : ffff8000101f50c4: add x0, x29, #0x200 : int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, }; 0.00 : ffff8000101f50c8: str xzr, [x29, #160] : int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 0.00 : ffff8000101f50cc: stp xzr, xzr, [x29, #168] 0.00 : ffff8000101f50d0: stp xzr, xzr, [x29, #184] : int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 0.00 : ffff8000101f50d4: stp xzr, xzr, [x0, #-248] 0.00 : ffff8000101f50d8: stp xzr, xzr, [x0, #-232] 0.00 : ffff8000101f50dc: stp xzr, xzr, [x0, #-216] 0.00 : ffff8000101f50e0: add x0, x29, #0x230 : int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 0.00 : ffff8000101f50e4: stp xzr, xzr, [x29, #200] : int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 0.00 : ffff8000101f50e8: stp xzr, xzr, [x29, #216] 0.00 : ffff8000101f50ec: stp xzr, xzr, [x29, #232] 0.00 : ffff8000101f50f0: stp xzr, xzr, [x29, #248] 0.00 : ffff8000101f50f4: stp xzr, xzr, [x0, #-248] 0.00 : ffff8000101f50f8: stp xzr, xzr, [x0, #-232] : int changes = 0; : : for_each_populated_zone(zone) { 0.00 : ffff8000101f50fc: bl ffff8000101f3c20 0.00 : ffff8000101f5100: cbz x0, ffff8000101f5330 0.00 : ffff8000101f5104: mov x21, x0 0.00 : ffff8000101f5108: add x22, x29, #0xa8 : numa_node_id(): : : #ifndef numa_node_id : /* Returns the number of the current Node. */ : static inline int numa_node_id(void) : { : return raw_cpu_read(numa_node); 0.00 : ffff8000101f510c: adrp x0, ffff800011526000 0.00 : ffff8000101f5110: add x26, x29, #0x90 0.00 : ffff8000101f5114: add x0, x0, #0xd58 : __xchg_case_8(): : : cl); \ : \ : return ret; \ : } : : __XCHG_CASE(w, b, , 8, , , , , , ) 0.00 : ffff8000101f5118: mov w19, #0x0 // #0 : refresh_cpu_vm_stats(): : : atomic_long_add(v, &zone->vm_stat[i]); : global_zone_diff[i] += v; : #ifdef CONFIG_NUMA : /* 3 seconds idle till flush */ : __this_cpu_write(p->expire, 3); 0.00 : ffff8000101f511c: mov w25, #0x3 // #3 : numa_node_id(): 0.00 : ffff8000101f5120: str x0, [x29, #104] : refresh_cpu_vm_stats(): : int changes = 0; 0.00 : ffff8000101f5124: str wzr, [x29, #112] 0.00 : ffff8000101f5128: b ffff8000101f513c : for_each_populated_zone(zone) { 0.00 : ffff8000101f512c: mov x0, x21 0.00 : ffff8000101f5130: bl ffff8000101f3cb8 0.00 : ffff8000101f5134: mov x21, x0 0.00 : ffff8000101f5138: cbz x0, ffff8000101f533c 0.00 : ffff8000101f513c: ldr x0, [x21, #120] 0.00 : ffff8000101f5140: cbz x0, ffff8000101f512c : struct per_cpu_pageset __percpu *p = zone->pageset; 0.00 : ffff8000101f5144: ldr x0, [x21, #88] 0.00 : ffff8000101f5148: add x27, x21, #0x5c0 0.00 : ffff8000101f514c: str x0, [x29, #120] 0.00 : ffff8000101f5150: mov x28, #0x0 // #0 0.00 : ffff8000101f5154: add x20, x0, #0x4f : __this_cpu_write(p->expire, 3); 0.00 : ffff8000101f5158: add x0, x0, #0x40 0.00 : ffff8000101f515c: str x0, [x29, #136] : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000101f5160: mrs x1, sp_el0 : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff8000101f5164: ldr w0, [x1, #16] : __preempt_count_add(): : } : : static inline void __preempt_count_add(int val) : { : u32 pc = READ_ONCE(current_thread_info()->preempt.count); : pc += val; 0.00 : ffff8000101f5168: add w0, w0, #0x1 : __write_once_size(): : static __always_inline void __write_once_size(volatile void *p, void *res, int size) : { : switch (size) { : case 1: *(volatile __u8 *)p = *(__u8 *)res; break; : case 2: *(volatile __u16 *)p = *(__u16 *)res; break; : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101f516c: str w0, [x1, #16] : refresh_cpu_vm_stats(): : v = this_cpu_xchg(p->vm_stat_diff[i], 0); 0.00 : ffff8000101f5170: add x0, x20, x28 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 0.00 : ffff8000101f5174: mrs x2, tpidr_el1 : __xchg_case_8(): 0.00 : ffff8000101f5178: add x0, x0, x2 0.00 : ffff8000101f517c: prfm pstl1strm, [x0] 0.00 : ffff8000101f5180: ldxrb w24, [x0] 0.00 : ffff8000101f5184: stxrb w4, w19, [x0] 0.00 : ffff8000101f5188: cbnz w4, ffff8000101f5180 : refresh_cpu_vm_stats(): 0.00 : ffff8000101f518c: sxtb w23, w24 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101f5190: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : { : struct thread_info *ti = current_thread_info(); : u64 pc = READ_ONCE(ti->preempt_count); : : /* Update only the count field, leaving need_resched unchanged */ : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000101f5194: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101f5198: str w0, [x1, #16] : __preempt_count_dec_and_test(): : * need of a reschedule. Otherwise, we need to reload the : * preempt_count in case the need_resched flag was cleared by an : * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE : * pair. : */ : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000101f519c: cbz x0, ffff8000101f52a8 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101f51a0: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000101f51a4: cbz x0, ffff8000101f52a8 : refresh_cpu_vm_stats(): : if (v) { 0.00 : ffff8000101f51a8: cbnz w23, ffff8000101f52b0 0.00 : ffff8000101f51ac: nop 0.00 : ffff8000101f51b0: add x28, x28, #0x1 : for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 0.00 : ffff8000101f51b4: cmp x28, #0xc 0.00 : ffff8000101f51b8: b.ne ffff8000101f5160 // b.any 0.00 : ffff8000101f51bc: ldr x0, [x29, #120] 0.00 : ffff8000101f51c0: add x1, x21, #0x620 0.00 : ffff8000101f51c4: mov x27, #0x0 // #0 : __xchg_case_16(): : __XCHG_CASE(w, h, , 16, , , , , , ) 0.00 : ffff8000101f51c8: mov w20, #0x0 // #0 0.00 : ffff8000101f51cc: add x24, x0, #0x42 : refresh_cpu_vm_stats(): : v = this_cpu_xchg(p->vm_numa_stat_diff[i], 0); : if (v) { : : atomic_long_add(v, &zone->vm_numa_stat[i]); : global_numa_diff[i] += v; : __this_cpu_write(p->expire, 3); 0.00 : ffff8000101f51d0: add x0, x0, #0x40 0.00 : ffff8000101f51d4: stp x0, x1, [x29, #128] : get_current(): 0.00 : ffff8000101f51d8: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000101f51dc: ldr w0, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000101f51e0: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101f51e4: str w0, [x1, #16] : refresh_cpu_vm_stats(): : v = this_cpu_xchg(p->vm_numa_stat_diff[i], 0); 0.00 : ffff8000101f51e8: add x0, x24, x27 : __my_cpu_offset(): 0.00 : ffff8000101f51ec: mrs x2, tpidr_el1 : __xchg_case_16(): 0.00 : ffff8000101f51f0: add x0, x0, x2 0.00 : ffff8000101f51f4: prfm pstl1strm, [x0] 0.00 : ffff8000101f51f8: ldxrh w23, [x0] 0.00 : ffff8000101f51fc: stxrh w5, w20, [x0] 0.00 : ffff8000101f5200: cbnz w5, ffff8000101f51f8 0.00 : ffff8000101f5204: and w28, w23, #0xffff : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101f5208: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000101f520c: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101f5210: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000101f5214: cbz x0, ffff8000101f52e0 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101f5218: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000101f521c: cbz x0, ffff8000101f52e0 : refresh_cpu_vm_stats(): : if (v) { 0.00 : ffff8000101f5220: cbnz w28, ffff8000101f52e8 0.00 : ffff8000101f5224: nop 0.00 : ffff8000101f5228: add x27, x27, #0x2 : for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) { 0.00 : ffff8000101f522c: cmp x27, #0xc 0.00 : ffff8000101f5230: b.ne ffff8000101f51d8 // b.any : } : } : : if (do_pagesets) { 0.00 : ffff8000101f5234: ldr w0, [x29, #116] 0.00 : ffff8000101f5238: cbz w0, ffff8000101f512c : * processor : * : * Check if there are pages remaining in this pageset : * if not then there is nothing to expire. : */ : if (!__this_cpu_read(p->expire) || 0.00 : ffff8000101f523c: ldr x23, [x29, #120] : __my_cpu_offset(): 0.00 : ffff8000101f5240: mrs x1, tpidr_el1 : refresh_cpu_vm_stats(): 0.00 : ffff8000101f5244: add x0, x23, #0x40 0.00 : ffff8000101f5248: ldrsb w2, [x0, x1] 0.00 : ffff8000101f524c: cbz w2, ffff8000101f512c 0.00 : ffff8000101f5250: ldr w2, [x23, x1] 0.00 : ffff8000101f5254: cbz w2, ffff8000101f512c : continue; : : /* : * We never drain zones local to this processor. : */ : if (zone_to_nid(zone) == numa_node_id()) { 0.00 : ffff8000101f5258: ldr w3, [x21, #72] : numa_node_id(): 0.00 : ffff8000101f525c: ldr x2, [x29, #104] : refresh_cpu_vm_stats(): 0.00 : ffff8000101f5260: ldr w2, [x2, x1] 0.00 : ffff8000101f5264: cmp w3, w2 0.00 : ffff8000101f5268: b.eq ffff8000101f5454 // b.none : __this_cpu_write(p->expire, 0); : continue; : } : : if (__this_cpu_dec_return(p->expire)) 0.00 : ffff8000101f526c: ldrb w2, [x0, x1] 0.00 : ffff8000101f5270: sub w2, w2, #0x1 0.00 : ffff8000101f5274: sxtb w2, w2 0.00 : ffff8000101f5278: strb w2, [x0, x1] 0.00 : ffff8000101f527c: cbnz w2, ffff8000101f512c : __my_cpu_offset(): 0.00 : ffff8000101f5280: mrs x1, tpidr_el1 : refresh_cpu_vm_stats(): : continue; : : if (__this_cpu_read(p->pcp.count)) { 0.00 : ffff8000101f5284: ldr w0, [x23, x1] 0.00 : ffff8000101f5288: cbz w0, ffff8000101f512c : drain_zone_pages(zone, this_cpu_ptr(&p->pcp)); : changes++; 0.00 : ffff8000101f528c: ldr w0, [x29, #112] : drain_zone_pages(zone, this_cpu_ptr(&p->pcp)); 0.00 : ffff8000101f5290: add x1, x23, x1 : changes++; 0.00 : ffff8000101f5294: add w0, w0, #0x1 0.00 : ffff8000101f5298: str w0, [x29, #112] : drain_zone_pages(zone, this_cpu_ptr(&p->pcp)); 0.00 : ffff8000101f529c: mov x0, x21 0.00 : ffff8000101f52a0: bl ffff8000102277b8 0.00 : ffff8000101f52a4: b ffff8000101f512c : v = this_cpu_xchg(p->vm_stat_diff[i], 0); 0.00 : ffff8000101f52a8: bl ffff800010cd2e78 : if (v) { 0.00 : ffff8000101f52ac: cbz w23, ffff8000101f51b0 : atomic_long_add(v, &zone->vm_stat[i]); 0.00 : ffff8000101f52b0: sxtb x3, w24 0.00 : ffff8000101f52b4: add x0, x27, x28, lsl #3 : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101f52b8: b ffff8000101f5320 0.00 : ffff8000101f52bc: b ffff8000101f5320 : __lse_atomic64_add(): : } : : ATOMIC64_OP(andnot, stclr) : ATOMIC64_OP(or, stset) : ATOMIC64_OP(xor, steor) : ATOMIC64_OP(add, stadd) 0.00 : ffff8000101f52c0: stadd x3, [x0] : refresh_cpu_vm_stats(): : global_zone_diff[i] += v; 0.00 : ffff8000101f52c4: ldr w1, [x22, x28, lsl #2] : __my_cpu_offset(): 0.00 : ffff8000101f52c8: mrs x3, tpidr_el1 : refresh_cpu_vm_stats(): 0.00 : ffff8000101f52cc: add w2, w1, w23 0.00 : ffff8000101f52d0: str w2, [x22, x28, lsl #2] : __this_cpu_write(p->expire, 3); 0.00 : ffff8000101f52d4: ldr x0, [x29, #136] 0.00 : ffff8000101f52d8: strb w25, [x0, x3] 0.00 : ffff8000101f52dc: b ffff8000101f51b0 : v = this_cpu_xchg(p->vm_numa_stat_diff[i], 0); 0.00 : ffff8000101f52e0: bl ffff800010cd2e78 : if (v) { 0.00 : ffff8000101f52e4: cbz w28, ffff8000101f5228 0.00 : ffff8000101f52e8: ldr x0, [x29, #136] : atomic_long_add(v, &zone->vm_numa_stat[i]); 0.00 : ffff8000101f52ec: and x4, x23, #0xffff 0.00 : ffff8000101f52f0: add x0, x0, x27, lsl #2 : arch_static_branch_jump(): 0.00 : ffff8000101f52f4: b ffff8000101f5328 0.00 : ffff8000101f52f8: b ffff8000101f5328 : __lse_atomic64_add(): 0.00 : ffff8000101f52fc: stadd x4, [x0] 0.00 : ffff8000101f5300: lsl x4, x27, #1 : refresh_cpu_vm_stats(): : __this_cpu_write(p->expire, 3); 0.00 : ffff8000101f5304: ldr x0, [x29, #128] : __my_cpu_offset(): 0.00 : ffff8000101f5308: mrs x5, tpidr_el1 : refresh_cpu_vm_stats(): : global_numa_diff[i] += v; 0.00 : ffff8000101f530c: ldr w1, [x26, x4] 0.00 : ffff8000101f5310: add w2, w1, w28 0.00 : ffff8000101f5314: str w2, [x26, x4] : __this_cpu_write(p->expire, 3); 0.00 : ffff8000101f5318: strb w25, [x0, x5] 0.00 : ffff8000101f531c: b ffff8000101f5228 : __ll_sc_atomic64_add(): : ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ : ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) : : ATOMIC64_OPS(add, add, I) 0.00 : ffff8000101f5320: b ffff8000101f76a0 0.00 : ffff8000101f5324: b ffff8000101f52c4 0.00 : ffff8000101f5328: b ffff8000101f76b8 0.00 : ffff8000101f532c: b ffff8000101f5300 0.00 : ffff8000101f5330: add x22, x29, #0xa8 0.00 : ffff8000101f5334: add x26, x29, #0x90 : refresh_cpu_vm_stats(): : int changes = 0; 0.00 : ffff8000101f5338: str wzr, [x29, #112] : } : } : #endif : } : : for_each_online_pgdat(pgdat) { 0.00 : ffff8000101f533c: bl ffff8000101f3c20 0.00 : ffff8000101f5340: add x21, x29, #0xd8 0.00 : ffff8000101f5344: mov x24, x0 : __xchg_case_8(): : __XCHG_CASE(w, b, , 8, , , , , , ) 0.00 : ffff8000101f5348: mov w19, #0x0 // #0 : refresh_cpu_vm_stats(): 0.00 : ffff8000101f534c: cbz x0, ffff8000101f53d4 : struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 0.00 : ffff8000101f5350: ldr x20, [x24, #7616] 0.00 : ffff8000101f5354: mov x0, #0x1dc8 // #7624 0.00 : ffff8000101f5358: add x23, x24, x0 0.00 : ffff8000101f535c: mov x27, #0x0 // #0 0.00 : ffff8000101f5360: add x20, x20, #0x1 0.00 : ffff8000101f5364: nop : get_current(): 0.00 : ffff8000101f5368: mrs x1, sp_el0 : __read_once_size(): 0.00 : ffff8000101f536c: ldr w0, [x1, #16] : __preempt_count_add(): : pc += val; 0.00 : ffff8000101f5370: add w0, w0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101f5374: str w0, [x1, #16] : refresh_cpu_vm_stats(): : : for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { : int v; : : v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 0.00 : ffff8000101f5378: add x0, x20, x27 : __my_cpu_offset(): 0.00 : ffff8000101f537c: mrs x2, tpidr_el1 : __xchg_case_8(): 0.00 : ffff8000101f5380: add x0, x0, x2 0.00 : ffff8000101f5384: prfm pstl1strm, [x0] 100.00 : ffff8000101f5388: ldxrb w28, [x0] 0.00 : ffff8000101f538c: stxrb w3, w19, [x0] 0.00 : ffff8000101f5390: cbnz w3, ffff8000101f5388 : refresh_cpu_vm_stats(): 0.00 : ffff8000101f5394: sxtb w25, w28 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101f5398: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): : WRITE_ONCE(ti->preempt.count, --pc); 0.00 : ffff8000101f539c: sub x0, x0, #0x1 : __write_once_size(): : case 4: *(volatile __u32 *)p = *(__u32 *)res; break; 0.00 : ffff8000101f53a0: str w0, [x1, #16] : __preempt_count_dec_and_test(): : return !pc || !READ_ONCE(ti->preempt_count); 0.00 : ffff8000101f53a4: cbz x0, ffff8000101f5420 : __read_once_size(): : __READ_ONCE_SIZE; 0.00 : ffff8000101f53a8: ldr x0, [x1, #16] : __preempt_count_dec_and_test(): 0.00 : ffff8000101f53ac: cbz x0, ffff8000101f5420 : refresh_cpu_vm_stats(): : if (v) { 0.00 : ffff8000101f53b0: cbnz w25, ffff8000101f5428 0.00 : ffff8000101f53b4: nop 0.00 : ffff8000101f53b8: add x27, x27, #0x1 : for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 0.00 : ffff8000101f53bc: cmp x27, #0x20 0.00 : ffff8000101f53c0: b.ne ffff8000101f5368 // b.any : for_each_online_pgdat(pgdat) { 0.00 : ffff8000101f53c4: mov x0, x24 0.00 : ffff8000101f53c8: bl ffff8000101f3c60 0.00 : ffff8000101f53cc: mov x24, x0 0.00 : ffff8000101f53d0: cbnz x0, ffff8000101f5350 : } : } : } : : #ifdef CONFIG_NUMA : changes += fold_diff(global_zone_diff, global_numa_diff, 0.00 : ffff8000101f53d4: mov x2, x21 0.00 : ffff8000101f53d8: mov x1, x26 0.00 : ffff8000101f53dc: mov x0, x22 0.00 : ffff8000101f53e0: bl ffff8000101f4f78 : global_node_diff); : #else : changes += fold_diff(global_zone_diff, global_node_diff); : #endif : return changes; : } 0.00 : ffff8000101f53e4: ldr w2, [x29, #112] 0.00 : ffff8000101f53e8: adrp x1, ffff800011909000 0.00 : ffff8000101f53ec: add x1, x1, #0x908 0.00 : ffff8000101f53f0: add w0, w0, w2 0.00 : ffff8000101f53f4: ldr x2, [x29, #344] 0.00 : ffff8000101f53f8: ldr x1, [x1] 0.00 : ffff8000101f53fc: eor x1, x2, x1 0.00 : ffff8000101f5400: cbnz x1, ffff8000101f545c 0.00 : ffff8000101f5404: ldp x19, x20, [sp, #16] 0.00 : ffff8000101f5408: ldp x21, x22, [sp, #32] 0.00 : ffff8000101f540c: ldp x23, x24, [sp, #48] 0.00 : ffff8000101f5410: ldp x25, x26, [sp, #64] 0.00 : ffff8000101f5414: ldp x27, x28, [sp, #80] 0.00 : ffff8000101f5418: ldp x29, x30, [sp], #352 0.00 : ffff8000101f541c: ret : v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 0.00 : ffff8000101f5420: bl ffff800010cd2e78 : if (v) { 0.00 : ffff8000101f5424: cbz w25, ffff8000101f53b8 : atomic_long_add(v, &pgdat->vm_stat[i]); 0.00 : ffff8000101f5428: sxtb x28, w28 0.00 : ffff8000101f542c: add x0, x23, x27, lsl #3 : arch_static_branch_jump(): 0.00 : ffff8000101f5430: b ffff8000101f544c 0.00 : ffff8000101f5434: b ffff8000101f544c : __lse_atomic64_add(): 0.00 : ffff8000101f5438: stadd x28, [x0] : refresh_cpu_vm_stats(): : global_node_diff[i] += v; 0.00 : ffff8000101f543c: ldr w0, [x21, x27, lsl #2] 0.00 : ffff8000101f5440: add w2, w0, w25 0.00 : ffff8000101f5444: str w2, [x21, x27, lsl #2] 0.00 : ffff8000101f5448: b ffff8000101f53b8 : __ll_sc_atomic64_add(): 0.00 : ffff8000101f544c: b ffff8000101f76d0 0.00 : ffff8000101f5450: b ffff8000101f543c : refresh_cpu_vm_stats(): : __this_cpu_write(p->expire, 0); 0.00 : ffff8000101f5454: strb wzr, [x0, x1] : continue; 0.00 : ffff8000101f5458: b ffff8000101f512c : } 0.00 : ffff8000101f545c: bl ffff8000100e64d0 <__stack_chk_fail> Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101caa40 : : perf_event_delayed_put(): : void perf_event_delayed_put(struct task_struct *task) : { : int ctxn; : : for_each_task_context_nr(ctxn) : WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 100.00 : ffff8000101caa40: ldr x1, [x0, #2104] 0.00 : ffff8000101caa44: cbnz x1, ffff8000101caa5c 0.00 : ffff8000101caa48: ldr x0, [x0, #2112] 0.00 : ffff8000101caa4c: cbnz x0, ffff8000101caa54 : } 0.00 : ffff8000101caa50: ret : WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 0.00 : ffff8000101caa54: brk #0x800 : } 0.00 : ffff8000101caa58: ret : WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 0.00 : ffff8000101caa5c: brk #0x800 0.00 : ffff8000101caa60: b ffff8000101caa48 Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff800010104b20 <__queue_work>: : __queue_work(): : return new_cpu; : } : : static void __queue_work(int cpu, struct workqueue_struct *wq, : struct work_struct *work) : { 0.00 : ffff800010104b20: stp x29, x30, [sp, #-112]! 0.00 : ffff800010104b24: mov x29, sp 0.00 : ffff800010104b28: str x21, [sp, #32] 0.00 : ffff800010104b2c: mov x21, x1 0.00 : ffff800010104b30: str x25, [sp, #64] 0.00 : ffff800010104b34: mov w25, w0 0.00 : ffff800010104b38: str x28, [sp, #88] 0.00 : ffff800010104b3c: mov x28, x2 : lockdep_assert_irqs_disabled(); : : debug_work_activate(work); : : /* if draining, only works from the same workqueue are allowed */ : if (unlikely(wq->flags & __WQ_DRAINING) && 0.00 : ffff800010104b40: ldr w0, [x1, #256] 0.00 : ffff800010104b44: tbnz w0, #16, ffff800010104dcc <__queue_work+0x2ac> 0.00 : ffff800010104b48: stp x19, x20, [x29, #16] : hash_64_generic(): : #endif : static __always_inline u32 hash_64_generic(u64 val, unsigned int bits) : { : #if BITS_PER_LONG == 64 : /* 64x64-bit multiply is efficient on all 64-bit processors */ : return val * GOLDEN_RATIO_64 >> (64 - bits); 0.00 : ffff800010104b4c: mov x20, #0x83eb // #33771 0.00 : ffff800010104b50: movk x20, #0x80b5, lsl #16 0.00 : ffff800010104b54: stp x22, x23, [x29, #40] 0.00 : ffff800010104b58: movk x20, #0x8646, lsl #32 0.00 : ffff800010104b5c: str x24, [x29, #56] 0.00 : ffff800010104b60: movk x20, #0x61c8, lsl #48 0.00 : ffff800010104b64: stp x26, x27, [x29, #72] : wq_select_unbound_cpu(): : if (likely(!wq_debug_force_rr_cpu)) { 0.00 : ffff800010104b68: adrp x22, ffff800011af9000 : rcu_read_lock(): : * read-side critical sections may be preempted and they may also block, but : * only when acquiring spinlocks that are subject to priority inheritance. : */ : static __always_inline void rcu_read_lock(void) : { : __rcu_read_lock(); 0.00 : ffff800010104b6c: bl ffff80001015c1b0 <__rcu_read_lock> : hash_64_generic(): 0.00 : ffff800010104b70: mul x20, x28, x20 : wq_select_unbound_cpu(): 0.00 : ffff800010104b74: add x22, x22, #0x9a0 : __queue_work(): : WARN_ON_ONCE(!is_chained_work(wq))) : return; : rcu_read_lock(); : retry: : if (req_cpu == WORK_CPU_UNBOUND) : cpu = wq_select_unbound_cpu(raw_smp_processor_id()); 0.00 : ffff800010104b78: adrp x23, ffff80001151d000 : bitmap_empty(): : static inline int bitmap_empty(const unsigned long *src, unsigned nbits) : { : if (small_const_nbits(nbits)) : return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); : : return find_first_bit(src, nbits) == nbits; 0.00 : ffff800010104b7c: add x27, x22, #0x10 : __queue_work(): 0.00 : ffff800010104b80: add x23, x23, #0x18 0.00 : ffff800010104b84: adrp x26, ffff800011909000 : hash_64_generic(): 0.00 : ffff800010104b88: lsr x20, x20, #58 0.00 : ffff800010104b8c: mov w24, w25 : __queue_work(): : if (req_cpu == WORK_CPU_UNBOUND) 0.00 : ffff800010104b90: cmp w25, #0x100 0.00 : ffff800010104b94: b.eq ffff800010104d10 <__queue_work+0x1f0> // b.none : : /* pwq which will be used unless @work is executing elsewhere */ : if (!(wq->flags & WQ_UNBOUND)) 0.00 : ffff800010104b98: ldr w0, [x21, #256] 0.00 : ffff800010104b9c: add x1, x26, #0x928 0.00 : ffff800010104ba0: ldr x1, [x1, w24, sxtw #3] 0.00 : ffff800010104ba4: tbnz w0, #1, ffff800010104c64 <__queue_work+0x144> : pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); 0.00 : ffff800010104ba8: ldr x19, [x21, #264] 0.00 : ffff800010104bac: add x19, x19, x1 : /* : * If @work was previously on a different pool, it might still be : * running there, in which case the work needs to be queued on that : * pool to guarantee non-reentrancy. : */ : last_pool = get_work_pool(work); 0.00 : ffff800010104bb0: mov x0, x28 0.00 : ffff800010104bb4: bl ffff800010103328 : if (last_pool && last_pool != pwq->pool) { 0.00 : ffff800010104bb8: ldr x1, [x19] 0.00 : ffff800010104bbc: cbz x0, ffff800010104bc8 <__queue_work+0xa8> 0.00 : ffff800010104bc0: cmp x0, x1 0.00 : ffff800010104bc4: b.ne ffff800010104c88 <__queue_work+0x168> // b.any : spin_lock(): : raw_spin_lock_init(&(_lock)->rlock); \ : } while (0) : : static __always_inline void spin_lock(spinlock_t *lock) : { : raw_spin_lock(&lock->rlock); 0.00 : ffff800010104bc8: mov x0, x1 0.00 : ffff800010104bcc: bl ffff800010cd80f0 <_raw_spin_lock> : __queue_work(): : * refcnt is zero, repeat pwq selection. Note that pwqs never die : * without another pwq replacing it in the numa_pwq_tbl or while : * work items are executing on it, so the retrying is guaranteed to : * make forward-progress. : */ : if (unlikely(!pwq->refcnt)) { 0.00 : ffff800010104bd0: ldr w0, [x19, #24] 0.00 : ffff800010104bd4: cbz w0, ffff800010104cf4 <__queue_work+0x1d4> : __read_once_size(): : }) : : static __always_inline : void __read_once_size(const volatile void *p, void *res, int size) : { : __READ_ONCE_SIZE; 0.00 : ffff800010104bd8: mov x0, x28 0.00 : ffff800010104bdc: ldr x1, [x0, #8]! : __queue_work(): : } : : /* pwq determined, queue */ : trace_workqueue_queue_work(req_cpu, pwq, work); : : if (WARN_ON(!list_empty(&work->entry))) 0.00 : ffff800010104be0: cmp x0, x1 0.00 : ffff800010104be4: b.ne ffff800010104e08 <__queue_work+0x2e8> // b.any : goto out; : : pwq->nr_in_flight[pwq->work_color]++; 0.00 : ffff800010104be8: ldr w3, [x19, #16] 0.00 : ffff800010104bec: add x1, x19, w3, sxtw #2 : work_color_to_flags(): : return color << WORK_STRUCT_COLOR_SHIFT; 0.00 : ffff800010104bf0: lsl w3, w3, #4 : __queue_work(): : pwq->nr_in_flight[pwq->work_color]++; 0.00 : ffff800010104bf4: ldr w0, [x1, #28] 0.00 : ffff800010104bf8: add w0, w0, #0x1 0.00 : ffff800010104bfc: str w0, [x1, #28] : work_flags = work_color_to_flags(pwq->work_color); : : if (likely(pwq->nr_active < pwq->max_active)) { 0.00 : ffff800010104c00: ldp w0, w1, [x19, #88] 0.00 : ffff800010104c04: cmp w0, w1 0.00 : ffff800010104c08: b.ge ffff800010104e10 <__queue_work+0x2f0> // b.tcont : trace_workqueue_activate_work(work); : pwq->nr_active++; : worklist = &pwq->pool->worklist; 0.00 : ffff800010104c0c: ldr x1, [x19] : pwq->nr_active++; 0.00 : ffff800010104c10: add w0, w0, #0x1 0.00 : ffff800010104c14: str w0, [x19, #88] : worklist = &pwq->pool->worklist; 0.00 : ffff800010104c18: add x2, x1, #0x20 : __read_once_size(): 0.00 : ffff800010104c1c: ldr x0, [x1, #32] : __queue_work(): : if (list_empty(worklist)) 0.00 : ffff800010104c20: cmp x2, x0 0.00 : ffff800010104c24: b.eq ffff800010104dac <__queue_work+0x28c> // b.none : } else { : work_flags |= WORK_STRUCT_DELAYED; : worklist = &pwq->delayed_works; : } : : insert_work(pwq, work, worklist, work_flags); 0.00 : ffff800010104c28: mov x1, x28 0.00 : ffff800010104c2c: mov x0, x19 0.00 : ffff800010104c30: bl ffff8000101028d8 : spin_unlock(): : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ : } while (0) : : static __always_inline void spin_unlock(spinlock_t *lock) : { : raw_spin_unlock(&lock->rlock); 0.00 : ffff800010104c34: ldr x0, [x19] 0.00 : ffff800010104c38: bl ffff800010cd7c38 <_raw_spin_unlock> : rcu_read_unlock(): : static inline void rcu_read_unlock(void) : { : RCU_LOCKDEP_WARN(!rcu_is_watching(), : "rcu_read_unlock() used illegally while idle"); : __release(RCU); : __rcu_read_unlock(); 0.00 : ffff800010104c3c: bl ffff800010160d20 <__rcu_read_unlock> 0.00 : ffff800010104c40: ldr x24, [x29, #56] 0.00 : ffff800010104c44: ldp x19, x20, [x29, #16] 0.00 : ffff800010104c48: ldp x22, x23, [x29, #40] 0.00 : ffff800010104c4c: ldp x26, x27, [x29, #72] : __queue_work(): : : out: : spin_unlock(&pwq->pool->lock); : rcu_read_unlock(); : } 0.00 : ffff800010104c50: ldr x21, [sp, #32] 0.00 : ffff800010104c54: ldr x25, [sp, #64] 0.00 : ffff800010104c58: ldr x28, [sp, #88] 0.00 : ffff800010104c5c: ldp x29, x30, [sp], #112 0.00 : ffff800010104c60: ret : cpu_to_node(): : #endif : : #ifndef cpu_to_node : static inline int cpu_to_node(int cpu) : { : return per_cpu(numa_node, cpu); 0.00 : ffff800010104c64: adrp x0, ffff800011526000 0.00 : ffff800010104c68: add x0, x0, #0xd58 0.00 : ffff800010104c6c: ldr w0, [x0, x1] : unbound_pwq_by_node(): : if (unlikely(node == NUMA_NO_NODE)) 0.00 : ffff800010104c70: cmn w0, #0x1 0.00 : ffff800010104c74: b.eq ffff800010104dc4 <__queue_work+0x2a4> // b.none : return rcu_dereference_raw(wq->numa_pwq_tbl[node]); 0.00 : ffff800010104c78: sxtw x0, w0 0.00 : ffff800010104c7c: add x0, x0, #0x22 : __read_once_size(): 0.00 : ffff800010104c80: ldr x19, [x21, x0, lsl #3] 0.00 : ffff800010104c84: b ffff800010104bb0 <__queue_work+0x90> 0.00 : ffff800010104c88: str x0, [x29, #104] : spin_lock(): : raw_spin_lock(&lock->rlock); 0.00 : ffff800010104c8c: bl ffff800010cd80f0 <_raw_spin_lock> : find_worker_executing_work(): : hash_for_each_possible(pool->busy_hash, worker, hentry, 0.00 : ffff800010104c90: ldr x2, [x29, #104] 0.00 : ffff800010104c94: ubfiz x0, x20, #3, #6 0.00 : ffff800010104c98: add x0, x2, x0 0.00 : ffff800010104c9c: ldr x0, [x0, #152] 0.00 : ffff800010104ca0: cbnz x0, ffff800010104cb0 <__queue_work+0x190> 0.00 : ffff800010104ca4: b ffff800010104cdc <__queue_work+0x1bc> 0.00 : ffff800010104ca8: ldr x0, [x0] 0.00 : ffff800010104cac: cbz x0, ffff800010104cdc <__queue_work+0x1bc> : if (worker->current_work == work && 0.00 : ffff800010104cb0: ldr x1, [x0, #16] 0.00 : ffff800010104cb4: cmp x28, x1 0.00 : ffff800010104cb8: b.ne ffff800010104ca8 <__queue_work+0x188> // b.any 0.00 : ffff800010104cbc: ldr x3, [x0, #24] 0.00 : ffff800010104cc0: ldr x1, [x28, #24] 0.00 : ffff800010104cc4: cmp x3, x1 0.00 : ffff800010104cc8: b.ne ffff800010104ca8 <__queue_work+0x188> // b.any : __queue_work(): : if (worker && worker->current_pwq->wq == wq) { 0.00 : ffff800010104ccc: ldr x0, [x0, #32] 0.00 : ffff800010104cd0: ldr x1, [x0, #8] 0.00 : ffff800010104cd4: cmp x21, x1 0.00 : ffff800010104cd8: b.eq ffff800010104dbc <__queue_work+0x29c> // b.none : spin_unlock(): : raw_spin_unlock(&lock->rlock); 0.00 : ffff800010104cdc: mov x0, x2 0.00 : ffff800010104ce0: bl ffff800010cd7c38 <_raw_spin_unlock> : spin_lock(): : raw_spin_lock(&lock->rlock); 0.00 : ffff800010104ce4: ldr x0, [x19] 0.00 : ffff800010104ce8: bl ffff800010cd80f0 <_raw_spin_lock> : __queue_work(): : if (unlikely(!pwq->refcnt)) { 0.00 : ffff800010104cec: ldr w0, [x19, #24] 0.00 : ffff800010104cf0: cbnz w0, ffff800010104bd8 <__queue_work+0xb8> : if (wq->flags & WQ_UNBOUND) { 0.00 : ffff800010104cf4: ldr w0, [x21, #256] 0.00 : ffff800010104cf8: tbz w0, #1, ffff800010104e3c <__queue_work+0x31c> : spin_unlock(): : raw_spin_unlock(&lock->rlock); 0.00 : ffff800010104cfc: ldr x0, [x19] 0.00 : ffff800010104d00: bl ffff800010cd7c38 <_raw_spin_unlock> : cpu_relax(): : : unsigned long get_wchan(struct task_struct *p); : : static inline void cpu_relax(void) : { : asm volatile("yield" ::: "memory"); 0.00 : ffff800010104d04: yield : __queue_work(): : if (req_cpu == WORK_CPU_UNBOUND) 0.00 : ffff800010104d08: cmp w25, #0x100 0.00 : ffff800010104d0c: b.ne ffff800010104b98 <__queue_work+0x78> // b.any : wq_select_unbound_cpu(): : if (likely(!wq_debug_force_rr_cpu)) { 0.00 : ffff800010104d10: ldrb w1, [x22, #72] : __queue_work(): : cpu = wq_select_unbound_cpu(raw_smp_processor_id()); 0.00 : ffff800010104d14: mov x0, x23 : __my_cpu_offset(): : : /* : * We want to allow caching the value, so avoid using volatile and : * instead use a fake stack read to hazard against barrier(). : */ : asm(ALTERNATIVE("mrs %0, tpidr_el1", 100.00 : ffff800010104d18: mrs x2, tpidr_el1 : __queue_work(): 0.00 : ffff800010104d1c: ldr w24, [x0, x2] : wq_select_unbound_cpu(): : if (likely(!wq_debug_force_rr_cpu)) { 0.00 : ffff800010104d20: cbnz w1, ffff800010104e1c <__queue_work+0x2fc> : test_bit(): : * @nr: bit number to test : * @addr: Address to start counting from : */ : static inline int test_bit(int nr, const volatile unsigned long *addr) : { : return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); 0.00 : ffff800010104d24: add w0, w24, #0x3f 0.00 : ffff800010104d28: cmp w24, #0x0 0.00 : ffff800010104d2c: csel w0, w0, w24, lt // lt = tstop 0.00 : ffff800010104d30: asr w0, w0, #6 0.00 : ffff800010104d34: sxtw x0, w0 0.00 : ffff800010104d38: ldr x0, [x27, x0, lsl #3] 0.00 : ffff800010104d3c: lsr x0, x0, x24 : wq_select_unbound_cpu(): : if (cpumask_test_cpu(cpu, wq_unbound_cpumask)) 0.00 : ffff800010104d40: tbnz w0, #0, ffff800010104b98 <__queue_work+0x78> : bitmap_empty(): 0.00 : ffff800010104d44: mov x2, #0x0 // #0 0.00 : ffff800010104d48: mov x1, #0x100 // #256 0.00 : ffff800010104d4c: mov x0, x27 0.00 : ffff800010104d50: bl ffff80001048daa8 : wq_select_unbound_cpu(): : if (cpumask_empty(wq_unbound_cpumask)) 0.00 : ffff800010104d54: cmp x0, #0x100 0.00 : ffff800010104d58: b.eq ffff800010104b98 <__queue_work+0x78> // b.none : __my_cpu_offset(): 0.00 : ffff800010104d5c: mrs x1, tpidr_el1 : wq_select_unbound_cpu(): : new_cpu = __this_cpu_read(wq_rr_cpu_last); 0.00 : ffff800010104d60: adrp x19, ffff80001151e000 0.00 : ffff800010104d64: add x0, x19, #0xfa8 : new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask); 0.00 : ffff800010104d68: ldr w0, [x0, x1] 0.00 : ffff800010104d6c: adrp x3, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010104d70: add x3, x3, #0x160 0.00 : ffff800010104d74: mov x1, x27 0.00 : ffff800010104d78: mov x2, x3 0.00 : ffff800010104d7c: str x3, [x29, #104] 0.00 : ffff800010104d80: bl ffff800010cb93f0 : if (unlikely(new_cpu >= nr_cpu_ids)) { 0.00 : ffff800010104d84: adrp x4, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010104d88: ldr x3, [x29, #104] 0.00 : ffff800010104d8c: ldr w1, [x4, #752] 0.00 : ffff800010104d90: cmp w0, w1 0.00 : ffff800010104d94: b.cs ffff800010104e70 <__queue_work+0x350> // b.hs, b.nlast : __this_cpu_write(wq_rr_cpu_last, new_cpu); 0.00 : ffff800010104d98: add x1, x19, #0xfa8 : return new_cpu; 0.00 : ffff800010104d9c: mov w24, w0 : __my_cpu_offset(): 0.00 : ffff800010104da0: mrs x2, tpidr_el1 : wq_select_unbound_cpu(): : __this_cpu_write(wq_rr_cpu_last, new_cpu); 0.00 : ffff800010104da4: str w0, [x1, x2] 0.00 : ffff800010104da8: b ffff800010104b98 <__queue_work+0x78> : __queue_work(): : pwq->pool->watchdog_ts = jiffies; 0.00 : ffff800010104dac: adrp x0, ffff800011907000 0.00 : ffff800010104db0: ldr x0, [x0, #2432] 0.00 : ffff800010104db4: str x0, [x1, #24] 0.00 : ffff800010104db8: b ffff800010104c28 <__queue_work+0x108> : pwq = worker->current_pwq; 0.00 : ffff800010104dbc: mov x19, x0 0.00 : ffff800010104dc0: b ffff800010104bd0 <__queue_work+0xb0> : unbound_pwq_by_node(): : return wq->dfl_pwq; 0.00 : ffff800010104dc4: ldr x19, [x21, #160] 0.00 : ffff800010104dc8: b ffff800010104bb0 <__queue_work+0x90> : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff800010104dcc: mrs x0, sp_el0 : __read_once_size(): 0.00 : ffff800010104dd0: ldr w1, [x0, #16] : current_wq_worker(): : /** : * current_wq_worker - return struct worker if %current is a workqueue worker : */ : static inline struct worker *current_wq_worker(void) : { : if (in_task() && (current->flags & PF_WQ_WORKER)) 0.00 : ffff800010104dd4: and w1, w1, #0x1fff00 0.00 : ffff800010104dd8: and w1, w1, #0xffff01ff 0.00 : ffff800010104ddc: cbnz w1, ffff800010104e00 <__queue_work+0x2e0> 0.00 : ffff800010104de0: ldr w1, [x0, #44] 0.00 : ffff800010104de4: tbz w1, #5, ffff800010104e00 <__queue_work+0x2e0> : return kthread_data(current); 0.00 : ffff800010104de8: bl ffff80001010d2a8 : is_chained_work(): : return worker && worker->current_pwq->wq == wq; 0.00 : ffff800010104dec: cbz x0, ffff800010104e00 <__queue_work+0x2e0> 0.00 : ffff800010104df0: ldr x0, [x0, #32] 0.00 : ffff800010104df4: ldr x0, [x0, #8] 0.00 : ffff800010104df8: cmp x21, x0 0.00 : ffff800010104dfc: b.eq ffff800010104b48 <__queue_work+0x28> // b.none : __queue_work(): : WARN_ON_ONCE(!is_chained_work(wq))) 0.00 : ffff800010104e00: brk #0x800 0.00 : ffff800010104e04: b ffff800010104c50 <__queue_work+0x130> : if (WARN_ON(!list_empty(&work->entry))) 0.00 : ffff800010104e08: brk #0x800 0.00 : ffff800010104e0c: b ffff800010104c34 <__queue_work+0x114> : work_flags |= WORK_STRUCT_DELAYED; 0.00 : ffff800010104e10: orr w3, w3, #0x2 : worklist = &pwq->delayed_works; 0.00 : ffff800010104e14: add x2, x19, #0x60 0.00 : ffff800010104e18: b ffff800010104c28 <__queue_work+0x108> : wq_select_unbound_cpu(): : } else if (!printed_dbg_warning) { 0.00 : ffff800010104e1c: ldrb w0, [x22, #73] 0.00 : ffff800010104e20: cbnz w0, ffff800010104d44 <__queue_work+0x224> : pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n"); 0.00 : ffff800010104e24: adrp x0, ffff8000111c3000 0.00 : ffff800010104e28: add x0, x0, #0x290 0.00 : ffff800010104e2c: bl ffff80001014a85c : printed_dbg_warning = true; 0.00 : ffff800010104e30: mov w0, #0x1 // #1 0.00 : ffff800010104e34: strb w0, [x22, #73] 0.00 : ffff800010104e38: b ffff800010104d44 <__queue_work+0x224> : __queue_work(): : WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt", 0.00 : ffff800010104e3c: adrp x1, ffff800011ad8000 0.00 : ffff800010104e40: add x1, x1, #0x4b4 0.00 : ffff800010104e44: ldrb w0, [x1, #3] 0.00 : ffff800010104e48: cbnz w0, ffff800010104bd8 <__queue_work+0xb8> 0.00 : ffff800010104e4c: mov w2, #0x1 // #1 0.00 : ffff800010104e50: adrp x0, ffff8000111c3000 0.00 : ffff800010104e54: strb w2, [x1, #3] 0.00 : ffff800010104e58: add x0, x0, #0x2e0 0.00 : ffff800010104e5c: mov w2, w24 0.00 : ffff800010104e60: add x1, x21, #0xb0 0.00 : ffff800010104e64: bl ffff8000100e6128 <__warn_printk> 0.00 : ffff800010104e68: brk #0x800 0.00 : ffff800010104e6c: b ffff800010104bd8 <__queue_work+0xb8> : wq_select_unbound_cpu(): : new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask); 0.00 : ffff800010104e70: mov x1, x27 0.00 : ffff800010104e74: mov x2, x3 0.00 : ffff800010104e78: mov w0, #0xffffffff // #-1 0.00 : ffff800010104e7c: bl ffff800010cb93f0 : if (unlikely(new_cpu >= nr_cpu_ids)) 0.00 : ffff800010104e80: adrp x4, ffff80001190a000 <__per_cpu_offset+0x6d8> 0.00 : ffff800010104e84: ldr w1, [x4, #752] 0.00 : ffff800010104e88: cmp w0, w1 0.00 : ffff800010104e8c: b.cc ffff800010104d98 <__queue_work+0x278> // b.lo, b.ul, b.last 0.00 : ffff800010104e90: b ffff800010104b98 <__queue_work+0x78> Percent | Source code & Disassembly of vmlinux for cycles (1 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101bf740 : : put_ctx(): : kfree(ctx->task_ctx_data); : kfree(ctx); : } : : static void put_ctx(struct perf_event_context *ctx) : { 100.00 : ffff8000101bf740: stp x29, x30, [sp, #-32]! 0.00 : ffff8000101bf744: mov x29, sp 0.00 : ffff8000101bf748: str x19, [sp, #16] 0.00 : ffff8000101bf74c: mov x19, x0 : if (refcount_dec_and_test(&ctx->refcount)) { 0.00 : ffff8000101bf750: add x0, x0, #0xac : arch_static_branch_jump(): : } : : static __always_inline bool arch_static_branch_jump(struct static_key *key, : bool branch) : { : asm_volatile_goto( 0.00 : ffff8000101bf754: b ffff8000101bf7a8 0.00 : ffff8000101bf758: b ffff8000101bf7a8 : __lse_atomic_fetch_sub_release(): : return i; \ : } : : ATOMIC_FETCH_OP_SUB(_relaxed, ) : ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") : ATOMIC_FETCH_OP_SUB(_release, l, "memory") 0.00 : ffff8000101bf75c: mov w1, #0x1 // #1 0.00 : ffff8000101bf760: neg w1, w1 0.00 : ffff8000101bf764: ldaddl w1, w1, [x0] : refcount_sub_and_test(): : */ : static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r) : { : int old = atomic_fetch_sub_release(i, &r->refs); : : if (old == i) { 0.00 : ffff8000101bf768: cmp w1, #0x1 0.00 : ffff8000101bf76c: b.ne ffff8000101bf7bc // b.any : smp_acquire__after_ctrl_dep(); 0.00 : ffff8000101bf770: dmb ishld : put_ctx(): : if (ctx->parent_ctx) 0.00 : ffff8000101bf774: ldr x0, [x19, #200] 0.00 : ffff8000101bf778: cbnz x0, ffff8000101bf80c : put_ctx(ctx->parent_ctx); : if (ctx->task && ctx->task != TASK_TOMBSTONE) 0.00 : ffff8000101bf77c: ldr x1, [x19, #176] 0.00 : ffff8000101bf780: sub x0, x1, #0x1 0.00 : ffff8000101bf784: cmn x0, #0x3 0.00 : ffff8000101bf788: b.ls ffff8000101bf7d0 // b.plast : put_task_struct(ctx->task); : call_rcu(&ctx->rcu_head, free_ctx); 0.00 : ffff8000101bf78c: add x0, x19, #0xf0 0.00 : ffff8000101bf790: adrp x1, ffff8000101ba000 0.00 : ffff8000101bf794: add x1, x1, #0xda8 0.00 : ffff8000101bf798: bl ffff80001015f0c8 : } : } 0.00 : ffff8000101bf79c: ldr x19, [sp, #16] 0.00 : ffff8000101bf7a0: ldp x29, x30, [sp], #32 0.00 : ffff8000101bf7a4: ret : __ll_sc_atomic_fetch_sub_release(): : ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ : ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ : ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) : : ATOMIC_OPS(add, add, I) : ATOMIC_OPS(sub, sub, J) 0.00 : ffff8000101bf7a8: mov w2, #0x1 // #1 0.00 : ffff8000101bf7ac: add x5, x19, #0xac 0.00 : ffff8000101bf7b0: b ffff8000101caf80 : refcount_sub_and_test(): : if (old == i) { 0.00 : ffff8000101bf7b4: cmp w1, #0x1 0.00 : ffff8000101bf7b8: b.eq ffff8000101bf770 // b.none : return true; : } : : if (unlikely(old < 0 || old - i < 0)) 0.00 : ffff8000101bf7bc: cmp w1, #0x0 0.00 : ffff8000101bf7c0: b.le ffff8000101bf824 : put_ctx(): 0.00 : ffff8000101bf7c4: ldr x19, [sp, #16] 0.00 : ffff8000101bf7c8: ldp x29, x30, [sp], #32 0.00 : ffff8000101bf7cc: ret : arch_atomic_fetch_sub_release(): : ATOMIC_FETCH_OPS(atomic_fetch_andnot) : ATOMIC_FETCH_OPS(atomic_fetch_or) : ATOMIC_FETCH_OPS(atomic_fetch_xor) : ATOMIC_FETCH_OPS(atomic_fetch_add) : ATOMIC_FETCH_OPS(atomic_fetch_and) : ATOMIC_FETCH_OPS(atomic_fetch_sub) 0.00 : ffff8000101bf7d0: bl ffff8000101bac18 : put_task_struct(): : : extern void __put_task_struct(struct task_struct *t); : : static inline void put_task_struct(struct task_struct *t) : { : if (refcount_dec_and_test(&t->usage)) 0.00 : ffff8000101bf7d4: add x3, x1, #0x28 : arch_atomic_fetch_sub_release(): 0.00 : ffff8000101bf7d8: tst w0, #0xff 0.00 : ffff8000101bf7dc: b.ne ffff8000101bf838 // b.any : __ll_sc_atomic_fetch_sub_release(): 0.00 : ffff8000101bf7e0: mov w2, #0x1 // #1 0.00 : ffff8000101bf7e4: add x6, x1, #0x28 0.00 : ffff8000101bf7e8: b ffff8000101caf98 : refcount_sub_and_test(): : if (old == i) { 0.00 : ffff8000101bf7ec: cmp w0, #0x1 0.00 : ffff8000101bf7f0: b.eq ffff8000101bf848 // b.none : if (unlikely(old < 0 || old - i < 0)) 0.00 : ffff8000101bf7f4: cmp w0, #0x0 0.00 : ffff8000101bf7f8: b.gt ffff8000101bf78c : refcount_warn_saturate(r, REFCOUNT_SUB_UAF); 0.00 : ffff8000101bf7fc: mov w1, #0x3 // #3 0.00 : ffff8000101bf800: mov x0, x3 0.00 : ffff8000101bf804: bl ffff8000104915e0 0.00 : ffff8000101bf808: b ffff8000101bf78c : put_ctx(): : put_ctx(ctx->parent_ctx); 0.00 : ffff8000101bf80c: bl ffff8000101bf740 : if (ctx->task && ctx->task != TASK_TOMBSTONE) 0.00 : ffff8000101bf810: ldr x1, [x19, #176] 0.00 : ffff8000101bf814: sub x0, x1, #0x1 0.00 : ffff8000101bf818: cmn x0, #0x3 0.00 : ffff8000101bf81c: b.hi ffff8000101bf78c // b.pmore 0.00 : ffff8000101bf820: b ffff8000101bf7d0 : refcount_sub_and_test(): 0.00 : ffff8000101bf824: mov w1, #0x3 // #3 0.00 : ffff8000101bf828: bl ffff8000104915e0 : put_ctx(): : } 0.00 : ffff8000101bf82c: ldr x19, [sp, #16] 0.00 : ffff8000101bf830: ldp x29, x30, [sp], #32 0.00 : ffff8000101bf834: ret : __lse_atomic_fetch_sub_release(): 0.00 : ffff8000101bf838: mov w0, #0x1 // #1 0.00 : ffff8000101bf83c: neg w0, w0 0.00 : ffff8000101bf840: ldaddl w0, w0, [x3] 0.00 : ffff8000101bf844: b ffff8000101bf7ec : refcount_sub_and_test(): : smp_acquire__after_ctrl_dep(); 0.00 : ffff8000101bf848: dmb ishld : put_task_struct(): : __put_task_struct(t); 0.00 : ffff8000101bf84c: mov x0, x1 0.00 : ffff8000101bf850: bl ffff8000100e2d50 <__put_task_struct> 0.00 : ffff8000101bf854: b ffff8000101bf78c Percent | Source code & Disassembly of vmlinux for cycles (5 samples, percent: local period) -------------------------------------------------------------------------------------------------- : : : : Disassembly of section .text: : : ffff8000101bac30 : : event_function(): : event_f func; : void *data; : }; : : static int event_function(void *info) : { 0.00 : ffff8000101bac30: stp x29, x30, [sp, #-64]! 0.00 : ffff8000101bac34: mov x29, sp 0.00 : ffff8000101bac38: stp x19, x20, [sp, #16] 0.00 : ffff8000101bac3c: stp x21, x22, [sp, #32] 0.00 : ffff8000101bac40: mov x22, x0 0.00 : ffff8000101bac44: stp x23, x24, [sp, #48] : struct event_function_struct *efs = info; : struct perf_event *event = efs->event; 0.00 : ffff8000101bac48: ldr x24, [x0] : struct perf_event_context *ctx = event->ctx; 0.00 : ffff8000101bac4c: ldr x20, [x24, #544] : __get_cpu_context(): : return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 0.00 : ffff8000101bac50: ldr x0, [x20] 0.00 : ffff8000101bac54: ldr x19, [x0, #72] 0.00 : ffff8000101bac58: bl ffff8000101ba6d8 <__my_cpu_offset> 0.00 : ffff8000101bac5c: add x19, x19, x0 : perf_ctx_lock(): : raw_spin_lock(&cpuctx->ctx.lock); 0.00 : ffff8000101bac60: add x23, x19, #0x8 0.00 : ffff8000101bac64: mov x0, x23 : event_function(): : struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); : struct perf_event_context *task_ctx = cpuctx->task_ctx; 0.00 : ffff8000101bac68: ldr x21, [x19, #256] : perf_ctx_lock(): : raw_spin_lock(&cpuctx->ctx.lock); 0.00 : ffff8000101bac6c: bl ffff800010cd80f0 <_raw_spin_lock> : if (ctx) 0.00 : ffff8000101bac70: cbz x21, ffff8000101bac7c : raw_spin_lock(&ctx->lock); 0.00 : ffff8000101bac74: add x0, x21, #0x8 0.00 : ffff8000101bac78: bl ffff800010cd80f0 <_raw_spin_lock> : event_function(): : perf_ctx_lock(cpuctx, task_ctx); : /* : * Since we do the IPI call without holding ctx->lock things can have : * changed, double check we hit the task we set out to hit. : */ : if (ctx->task) { 0.00 : ffff8000101bac7c: ldr x0, [x20, #176] 0.00 : ffff8000101bac80: cbz x0, ffff8000101bace4 : get_current(): : */ : static __always_inline struct task_struct *get_current(void) : { : unsigned long sp_el0; : : asm ("mrs %0, sp_el0" : "=r" (sp_el0)); 0.00 : ffff8000101bac84: mrs x1, sp_el0 : event_function(): : if (ctx->task != current) { 0.00 : ffff8000101bac88: cmp x0, x1 0.00 : ffff8000101bac8c: b.ne ffff8000101bad04 // b.any : * and event_function() is only ever called when active (or : * rather, we'll have bailed in task_function_call() or the : * above ctx->task != current test), therefore we must have : * ctx->is_active here. : */ : WARN_ON_ONCE(!ctx->is_active); 0.00 : ffff8000101bac90: ldr w0, [x20, #152] 0.00 : ffff8000101bac94: cbz w0, ffff8000101bacf4 : /* : * And since we have ctx->is_active, cpuctx->task_ctx must : * match. : */ : WARN_ON_ONCE(task_ctx != ctx); 0.00 : ffff8000101bac98: cmp x20, x21 0.00 : ffff8000101bac9c: b.ne ffff8000101bacfc // b.any : } else { : WARN_ON_ONCE(&cpuctx->ctx != ctx); : } : : efs->func(event, cpuctx, ctx, efs->data); 0.00 : ffff8000101baca0: ldp x4, x3, [x22, #8] 0.00 : ffff8000101baca4: mov x1, x19 0.00 : ffff8000101baca8: mov x2, x20 0.00 : ffff8000101bacac: mov x0, x24 : int ret = 0; 0.00 : ffff8000101bacb0: mov w19, #0x0 // #0 : efs->func(event, cpuctx, ctx, efs->data); 0.00 : ffff8000101bacb4: blr x4 : perf_ctx_unlock(): : if (ctx) 0.00 : ffff8000101bacb8: cbz x21, ffff8000101bacc4 : raw_spin_unlock(&ctx->lock); 0.00 : ffff8000101bacbc: add x0, x21, #0x8 0.00 : ffff8000101bacc0: bl ffff800010cd7c38 <_raw_spin_unlock> : raw_spin_unlock(&cpuctx->ctx.lock); 0.00 : ffff8000101bacc4: mov x0, x23 0.00 : ffff8000101bacc8: bl ffff800010cd7c38 <_raw_spin_unlock> : event_function(): : unlock: : perf_ctx_unlock(cpuctx, task_ctx); : : return ret; : } 0.00 : ffff8000101baccc: mov w0, w19 0.00 : ffff8000101bacd0: ldp x19, x20, [sp, #16] 0.00 : ffff8000101bacd4: ldp x21, x22, [sp, #32] 100.00 : ffff8000101bacd8: ldp x23, x24, [sp, #48] 0.00 : ffff8000101bacdc: ldp x29, x30, [sp], #64 0.00 : ffff8000101bace0: ret : WARN_ON_ONCE(&cpuctx->ctx != ctx); 0.00 : ffff8000101bace4: cmp x19, x20 0.00 : ffff8000101bace8: b.eq ffff8000101baca0 // b.none 0.00 : ffff8000101bacec: brk #0x800 0.00 : ffff8000101bacf0: b ffff8000101baca0 : WARN_ON_ONCE(!ctx->is_active); 0.00 : ffff8000101bacf4: brk #0x800 0.00 : ffff8000101bacf8: b ffff8000101bac98 : WARN_ON_ONCE(task_ctx != ctx); 0.00 : ffff8000101bacfc: brk #0x800 0.00 : ffff8000101bad00: b ffff8000101baca0 : ret = -ESRCH; 0.00 : ffff8000101bad04: mov w19, #0xfffffffd // #-3 0.00 : ffff8000101bad08: b ffff8000101bacb8