背景
ubuntu机器发生了宕机，需要排查
内核版本是5.4.0-139
调查
首先先看log
[4580486.597245] watchdog: BUG: soft lockup - CPU#24 stuck for22s! [migration/24:156]
[4580486.605816] Modules linked in: cdc_ether usbnet mii mmfs26(OE)mmfslinux(OE)tracedev(OE)nvidia_uvm(OE)nvidia_peermem(POE) veth nvidia_drm(POE)nvidia_modeset(POE)nvidia(POE) ip6t_REJECT nf_reject_ipv6 nf_conntrack_netlink ipt_REJECT nf_reject_ipv4 xt_mark xt_addrtype xt_MASQUERADE xt_set ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ip ip_set_bitmap_port ip_set_hash_ipport dummy ip_set nf_tables nfnetlink ip_vs_wrr ip_vs_sh ip_vs_rr ip_vs ipvlan overlay iptable_nat ip6table_filter ip6table_nat nf_nat ip6table_mangle ip6_tables xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c xt_comment iptable_mangle iptable_filter bpfilter aufs msr cpuid xsk_diag tcp_diag udp_diag raw_diag inet_diag unix_diag af_packet_diag netlink_diag ib_ipoib(OE)ib_umad(OE)rdma_ucm(OE)rdma_cm(OE)iw_cm(OE)ib_cm(OE) nls_iso8859_1 ses enclosure intel_rapl_msr intel_rapl_common binfmt_misc nfit x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm crct10dif_pclmul crc32_pclmul
[4580486.605843] ghash_clmulni_intel aesni_intel crypto_simd ipmi_ssif cryptd glue_helper mpt3sas mxm_wmi raid_class video scsi_transport_sas input_leds joydev ast drm_vram_helper mlx5_ib(OE) ttm ib_uverbs(OE) drm_kms_helper isst_if_mmio isst_if_mbox_pci ib_core(OE) isst_if_common i2c_algo_bit ioatdma mei_me fb_sys_fops syscopyarea sysfillrect sysimgblt i2c_i801 mei dca ipmi_si ipmi_devintf ipmi_msghandler mac_hid acpi_power_meter acpi_pad sch_fq_codel ramoops reed_solomon efi_pstore drm ip_tables x_tables hid_generic mlx5_core(OE) nvme mlxfw(OE) pci_hyperv_intf nvme_core tls psample usbhid mlxdevm(OE) virtio_blk auxiliary(OE) hid mlx_compat(OE) ahci libahci wmi knem(OE) autofs4 [last unloaded: ecc]
[4580486.605865] CPU: 24 PID: 156 Comm: migration/24 Kdump: loaded Tainted: P OE 5.4.0-139-generic #156-Ubuntu
[4580486.605866] Hardware name: Nettrix SSNETTRIXHBX-GN6-F562/60WB32, BIOS BKMH011054-U08 11/29/2024
[4580486.605871] RIP: 0010:stop_machine_yield+0xc/0x10
[4580486.605873] Code: 00 00 00 75 0d 4c 8b 65 f8 c9 c3 b8 fe ff ff ff eb e3 e8 37 2b f3 ff 0f 1f 80 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 f3 90 5d <c3> 0f 1f 00 0f 1f 44 00 00 55 48 89 e5 41 57 49 89 ff 41 56 41 55
[4580486.605874] RSP: 0000:ffffabd919427e50 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13
[4580486.605876] RAX: 0000000000000282 RBX: ffffabd935427bc4 RCX: ffff935b3f623610
[4580486.605876] RDX: ffffabd935427b20 RSI: 0000000000000282 RDI: ffffffffa1a56d00
[4580486.605877] RBP: ffffabd919427e90 R08: 0000000000000084 R09: 0000000000000001
[4580486.605877] R10: 000000000af130c3 R11: ffff935b3f62ffb8 R12: 0000000000000001
[4580486.605877] R13: ffffffffa1a56d00 R14: 0000000000000001 R15: ffffabd935427ba0
[4580486.605878] FS: 0000000000000000(0000) GS:ffff935b3f600000(0000) knlGS:0000000000000000
[4580486.605879] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[4580486.605879] CR2: 000010b322096000 CR3: 000000e4907f0002 CR4: 0000000000760ee0
[4580486.605880] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[4580486.605880] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[4580486.605881] PKRU: 55555554
[4580486.605881] Call Trace:
[4580486.605884] ? multi_cpu_stop+0x9d/0x110
[4580486.605886] ? stop_machine_yield+0x10/0x10
[4580486.605888] cpu_stopper_thread+0x50/0x100
[4580486.605890] smpboot_thread_fn+0xd0/0x170
[4580486.605892] kthread+0x104/0x140
[4580486.605893] ? sort_range+0x30/0x30
[4580486.605894] ? kthread_park+0x90/0x90
[4580486.605896] ret_from_fork+0x1f/0x40
[4580486.605898] Kernel panic - not syncing: softlockup: hung tasks
[4580486.612805] CPU: 24 PID: 156 Comm: migration/24 Kdump: loaded Tainted: P OEL 5.4.0-139-generic #156-Ubuntu
[4580486.624653] Hardware name: Nettrix SSNETTRIXHBX-GN6-F562/60WB32, BIOS BKMH011054-U08 11/29/2024
[4580486.634400] Call Trace:
[4580486.637900] <IRQ>
[4580486.640982] dump_stack+0x6d/0x8b
[4580486.645335] panic+0x101/0x2e3
[4580486.649396] watchdog_timer_fn.cold+0x86/0xa0
[4580486.654791] __hrtimer_run_queues+0xf7/0x270
[4580486.660086] ? softlockup_fn+0x50/0x50
[4580486.664820] hrtimer_interrupt+0x109/0x220
[4580486.669866] smp_apic_timer_interrupt+0x71/0x140
[4580486.675405] apic_timer_interrupt+0xf/0x20
[4580486.680426] </IRQ>
[4580486.683443] RIP: 0010:stop_machine_yield+0xc/0x10
[4580486.689093] Code: 00 00 00 75 0d 4c 8b 65 f8 c9 c3 b8 fe ff ff ff eb e3 e8 37 2b f3 ff 0f 1f 80 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 f3 90 5d <c3> 0f 1f 00 0f 1f 44 00 00 55 48 89 e5 41 57 49 89 ff 41 56 41 55
[4580486.709905] RSP: 0000:ffffabd919427e50 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13
[4580486.718548] RAX: 0000000000000282 RBX: ffffabd935427bc4 RCX: ffff935b3f623610
[4580486.726772] RDX: ffffabd935427b20 RSI: 0000000000000282 RDI: ffffffffa1a56d00
[4580486.734952] RBP: ffffabd919427e90 R08: 0000000000000084 R09: 0000000000000001
[4580486.737246] watchdog: BUG: soft lockup - CPU#33 stuck for 22s! [node:3148320]
[4580486.743131] R10: 000000000af130c3 R11: ffff935b3f62ffb8 R12: 0000000000000001
[4580486.743134] R13: ffffffffa1a56d00 R14: 0000000000000001 R15: ffffabd935427ba0
[4580486.751175] Modules linked in: cdc_ether usbnet mii mmfs26(OE)mmfslinux(OE)tracedev(OE)nvidia_uvm(OE)nvidia_peermem(POE) veth nvidia_drm(POE)nvidia_modeset(POE)nvidia(POE) ip6t_REJECT nf_reject_ipv6 nf_conntrack_netlink ipt_REJECT nf_reject_ipv4 xt_mark xt_addrtype xt_MASQUERADE xt_set ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ip ip_set_bitmap_port ip_set_hash_ipport dummy ip_set nf_tables nfnetlink ip_vs_wrr ip_vs_sh ip_vs_rr ip_vs ipvlan overlay iptable_nat ip6table_filter ip6table_nat nf_nat ip6table_mangle ip6_tables xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c xt_comment iptable_mangle iptable_filter bpfilter aufs msr cpuid xsk_diag tcp_diag udp_diag raw_diag inet_diag unix_diag af_packet_diag netlink_diag ib_ipoib(OE)ib_umad(OE)rdma_ucm(OE)rdma_cm(OE)iw_cm(OE)ib_cm(OE) nls_iso8859_1 ses enclosure intel_rapl_msr intel_rapl_common binfmt_misc nfit x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm crct10dif_pclmul crc32_pclmul
[4580486.751204] ghash_clmulni_intel
[4580486.759358] ? multi_cpu_stop+0x9d/0x110
[4580486.767353] aesni_intel crypto_simd ipmi_ssif cryptd glue_helper mpt3sas mxm_wmi raid_class video scsi_transport_sas input_leds joydev ast drm_vram_helper mlx5_ib(OE) ttm ib_uverbs(OE) drm_kms_helper isst_if_mmio isst_if_mbox_pci ib_core(OE) isst_if_common i2c_algo_bit ioatdma mei_me fb_sys_fops syscopyarea sysfillrect sysimgblt i2c_i801 mei dca ipmi_si ipmi_devintf ipmi_msghandler mac_hid acpi_power_meter acpi_pad sch_fq_codel ramoops reed_solomon efi_pstore drm ip_tables x_tables hid_generic mlx5_core(OE) n
vme mlxfw(OE) pci_hyperv_intf nvme_core tls psample usbhid mlxdevm(OE) virtio_blk auxiliary(OE) hid mlx_compat(OE) ahci libahci wmi knem(OE) autofs4 [last unloaded: ecc]
[4580486.861255] ? stop_machine_yield+0x10/0x10
[4580486.865335] CPU: 33 PID: 3148320 Comm: node Kdump: loaded Tainted: P OEL 5.4.0-139-generic #156-Ubuntu
[4580486.870242] cpu_stopper_thread+0x50/0x100
[4580486.933988] Hardware name: Nettrix SSNETTRIXHBX-GN6-F562/60WB32, BIOS BKMH011054-U08 11/29/2024
[4580486.933997] RIP: 0010:__fsnotify_update_child_dentry_flags.part.0+0xbe/0x110
[4580486.939171] smpboot_thread_fn+0xd0/0x170
[4580486.950445] Code: eb 43 80 cc 40 41 89 07 4c 89 f7 c6 07 00 0f 1f 40 00 49 8b 87 90 00 00 00 4c 8d b8 70 ff ff ff 48 39 c3 74 20 49 83 7f 30 00 <74> e6 4d 8d 77 58 4c 89 f7 e8 94 70 7d 00 41 8b 07 45 85 ed 75 c2
[4580486.955540] kthread+0x104/0x140
[4580486.965066] RSP: 0018:ffffabd947c7fdc8 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13
[4580486.973074] ? sort_range+0x30/0x30
[4580486.977891] RAX: ffff92f234836090 RBX: ffff92fded6551e0 RCX: 0000000000000000
[4580486.977895] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff930e939c1918
[4580486.998388] ? kthread_park+0x90/0x90
[4580487.002448] RBP: ffffabd947c7fe00 R08: 0000000000000000 R09: 0000000000100000
[4580487.002450] R10: ffff93d42b4d4660 R11: 0000000000000210 R12: ffff92fded655140
[4580487.010972] ret_from_fork+0x1f/0x40
[4580487.015291] R13: 0000000000000006 R14: ffff930e939c1918 R15: ffff92f234836000
[4580487.015293] FS: 00007f34a76fe640(0000) GS:ffff93dade040000(0000) knlGS:0000000000000000language-c复制代码
这里可以看出是soft lockup了，谁导致的呢？怀疑的有两个cpu，一个24号上的migration任务，一个33号cpu上的node任务。看这个log有点乱了，不过大致还是能看出来，最后24号cpu的rsp是ffffabd919427e50，33号cpu的rsp是ffffabd919427e50。
- 24号cpu的migration进程
可以看到，只有24号cpu上的migration运行了26秒之多，这个肯定是异常的，26秒也足够导致softlockup了
crash> ps -m 12
[ 000:00:00.550] [IN] PID: 12 TASK: ffff935b319ebc00 CPU: 0 COMMAND: "migration/0"
crash> ps -m 18
[ 000:00:00.534] [IN] PID: 18 TASK: ffff935b30c0bc00 CPU: 1 COMMAND: "migration/1"
crash> ps -m 24
[ 000:00:00.530] [IN] PID: 24 TASK: ffff935b30c61e00 CPU: 2 COMMAND: "migration/2"
crash> ps -m 156
[ 000:00:26.293] [RU] PID: 156 TASK: ffff935b30a28000 CPU: 24 COMMAND: "migration/24"
crash> ps -m 180
[ 000:00:00.210] [IN] PID: 180 TASK: ffff935b30b79e00 CPU: 28 COMMAND: "migration/28"language-c复制代码
那么这个在做什么?
crash> bt
PID: 156 TASK: ffff935b30a28000 CPU: 24 COMMAND: "migration/24"
#0 [ffffabd919444d20] machine_kexec at ffffffffa0a6ca23
#1 [ffffabd919444d80] __crash_kexec at ffffffffa0b4cc42
#2 [ffffabd919444e50] panic at ffffffffa1497f8b
#3 [ffffabd919444ed0] watchdog_timer_fn.cold at ffffffffa14a0fd7
#4 [ffffabd919444f08] __hrtimer_run_queues at ffffffffa0b2b667
#5 [ffffabd919444f70] hrtimer_interrupt at ffffffffa0b2bf19
bt: invalid kernel virtual address: ffffabd919445000 type: "pt_regs"language-c复制代码
栈被破坏了？timer之前的内容没有了？
好在log中可以看到rsp
crash> bt -S ffffabd919427e50
PID: 156 TASK: ffff935b30a28000 CPU: 24 COMMAND: "migration/24"
#0 [ffffabd919427e50] __schedule at ffffffffa14f3393
#1 [ffffabd919427e50] multi_cpu_stop at ffffffffa0b67dad
#2 [ffffabd919427e98] cpu_stopper_thread at ffffffffa0b67bb0
#3 [ffffabd919427ed8] smpboot_thread_fn at ffffffffa0ac8280
#4 [ffffabd919427f08] kthread at ffffffffa0ac10e4
#5 [ffffabd919427f50] ret_from_fork at ffffffffa160023flanguage-c复制代码
看上去没什么异常的啊？
- 33号cpu的node进程
crash> bt
PID: 3148320 TASK: ffff92e0711f0000 CPU: 33 COMMAND: "node"
#0 [fffffe0000764e40] crash_nmi_callback at ffffffffa0a60387
#1 [fffffe0000764e50] nmi_handle at ffffffffa0a35cd1
#2 [fffffe0000764ea8] default_do_nmi at ffffffffa0a35e6a
#3 [fffffe0000764ed0] do_nmi at ffffffffa0a3609e
#4 [fffffe0000764ef0] end_repeat_nmi at ffffffffa1601788
[exception RIP: delay_tsc+54]
RIP: ffffffffa148d826 RSP: ffffabd919620c00 RFLAGS: 00000046
RAX: 0000000011b69f75 RBX: 0000000000002704 RCX: 0000000000001021
RDX: 00000000005748f7 RSI: 0000000000000021 RDI: 0000000000000a15
RBP: ffffabd919620c00 R8: 005748f711b695ed R9: 0000000000000021
R10: ffffffffa29a0228 R11: ffffabd919620cd0 R12: 0000000000000020
R13: ffffffffa2b6a7a0 R14: ffffffffa29a0224 R15: ffffffffa2b6a7a0
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
--- <NMI exception stack> ---
#5 [ffffabd919620c00] delay_tsc at ffffffffa148d826
#6 [ffffabd919620c08] __const_udelay at ffffffffa148d7a3
#7 [ffffabd919620c18] wait_for_xmitr at ffffffffa10959da
#8 [ffffabd919620c40] serial8250_console_putchar at ffffffffa1095a7d
#9 [ffffabd919620c60] uart_console_write at ffffffffa108e0fc
#10 [ffffabd919620c90] serial8250_console_write at ffffffffa1099c01
#11 [ffffabd919620d10] univ8250_console_write at ffffffffa1093646
#12 [ffffabd919620d20] console_unlock at ffffffffa0b08092
#13 [ffffabd919620d68] vprintk_emit at ffffffffa0b09b5e
#14 [ffffabd919620db8] vprintk_default at ffffffffa0b09c99
#15 [ffffabd919620dc8] vprintk_func at ffffffffa0b0a76c
#16 [ffffabd919620de8] printk at ffffffffa149d8c1
#17 [ffffabd919620e48] __show_regs.cold at ffffffffa1491329
#18 [ffffabd919620eb8] show_regs at ffffffffa0a357e4
#19 [ffffabd919620ed0] watchdog_timer_fn.cold at ffffffffa14a0f98
#20 [ffffabd919620f08] __hrtimer_run_queues at ffffffffa0b2b667
#21 [ffffabd919620f70] hrtimer_interrupt at ffffffffa0b2bf19
bt: invalid kernel virtual address: ffffabd919621000 type: "pt_regs"language-c复制代码
这里栈也损坏了，看不到之前的了，这个只能看出来，timer来了之后的栈，之前呢？还好log中能看到33号cpu之前的rsp
crash> bt -S ffffabd947c7fdc8
PID: 3148320 TASK: ffff92e0711f0000 CPU: 33 COMMAND: "node"
#0 [ffffabd947c7fdc8] __schedule at ffffffffa14f3393
#1 [ffffabd947c7fe08] __fsnotify_update_child_dentry_flags at ffffffffa0d2191b
#2 [ffffabd947c7fe18] fsnotify_recalc_mask at ffffffffa0d226f8
#3 [ffffabd947c7fe30] fsnotify_add_mark_locked at ffffffffa0d22d1e
#4 [ffffabd947c7fe90] inotify_update_watch at ffffffffa0d2499f
#5 [ffffabd947c7fee0] __x64_sys_inotify_add_watch at ffffffffa0d24c28
#6 [ffffabd947c7ff30] do_syscall_64 at ffffffffa0a04fd7
#7 [ffffabd947c7ff50] entry_SYSCALL_64_after_hwframe at ffffffffa16000a4
RIP: 00007f34ae226c0b RSP: 00007f34a76fdbb8 RFLAGS: 00000206
RAX: ffffffffffffffda RBX: 00007f349420c260 RCX: 00007f34ae226c0b
RDX: 0000000007000fc6 RSI: 00007f349420c188 RDI: 0000000000000015
RBP: 00007f34a76fdc10 R8: 000000000000021d R9: 000000000000013d
R10: 00007f3494338e50 R11: 0000000000000206 R12: 00007f34a76fdc68
R13: 00007f349420c270 R14: 0000000006fc7320 R15: 00007f34a76fdc80
ORIG_RAX: 00000000000000fe CS: 0033 SS: 002b
crash> timer -C 33
JIFFIES
5440018529
TIMER_BASES[33][BASE_STD]: ffff93dade060a80
EXPIRES TTE TIMER_LIST FUNCTION
5440015122-3407 ffff93561bfb9448 ffffffffa0aba7e0 <delayed_work_timer_fn>
5440015335-3194 ffff93d855b46650 ffffffffa130b3f0 <neigh_timer_handler>
5440015429-3100 ffff932e34d61020 ffffffffa1381b40 <tw_timer_handler>
5440015431-3098 ffff932e34d62760 ffffffffa1381b40 <tw_timer_handler>
5440015434-3095 ffff932e34d60d38 ffffffffa1381b40 <tw_timer_handler>
5440015437-3092 ffff932e34d61400 ffffffffa1381b40 <tw_timer_handler>
5440015439-3090 ffff932e34d60198 ffffffffa1381b40 <tw_timer_handler>
5440016478-2051 ffff93bed3947358 ffffffffa139db50 <tcp_keepalive_timer>
5440016813-1716 ffff9356caa0f448 ffffffffa0aba7e0 <delayed_work_timer_fn>
5440016813-1716 ffff935b29594448 ffffffffa0aba7e0 <delayed_work_timer_fn>
5440017259-1270 ffff935aa4a97448 ffffffffa0aba7e0 <delayed_work_timer_fn>
5440017375-1154 ffff9356a830c448 ffffffffa0aba7e0 <delayed_work_timer_fn>
5440017375-1154 ffff93d733914448 ffffffffa0aba7e0 <delayed_work_timer_fn>
5440017638-891 ffff935b2aa44448 ffffffffa0aba7e0 <delayed_work_timer_fn>
5440017901-628 ffff935b09e00480 ffffffffa1354050 <dev_watchdog>
544007515156622 ffff93dade05bc20 ffffffffa0a4d0e0 <mce_timer_fn>
5440240252221723 ffff93d7ed8bca60 ffffffffc0a62bd0 <ip_vs_conn_expire>
TIMER_BASES[33][BASE_DEF]: ffff93dade061d00
EXPIRES TTE TIMER_LIST FUNCTION
544004336524836 ffff93dade06f848 ffffffffa0ab9cc0 <idle_worker_timeout>language-c复制代码
那就看看__fsnotify_update_child_dentry_flags函数？
void __fsnotify_update_child_dentry_flags(struct inode *inode)
{
structdentry *alias;
int watched;
if (!S_ISDIR(inode->i_mode))
return;
/* determine if the children should tell inode about their events */
watched = fsnotify_inode_watches_children(inode);
spin_lock(&inode->i_lock);
/* run all of the dentries associated with this inode. Since this is a
* directory, there damn well better only be one item on this list */
hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
structdentry *child;
/* run all of the children of the original inode and fix their
* d_flags to indicate parental interest (their parent is the
* original inode) */
spin_lock(&alias->d_lock);
list_for_each_entry(child, &alias->d_subdirs, d_child) {
if (!child->d_inode)
continue;
spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED);
if (watched)
child->d_flags |= DCACHE_FSNOTIFY_PARENT_WATCHED;
else
child->d_flags &= ~DCACHE_FSNOTIFY_PARENT_WATCHED;
spin_unlock(&child->d_lock);
}
spin_unlock(&alias->d_lock);
}
spin_unlock(&inode->i_lock);
}language-c复制代码
在锁里执行时间太久了？
搜一下这个栈，看上去和一个已知问题比较像，https://access.redhat.com/solutions/7095274
那跟着大佬的思路看看是不是一个问题呢？
//主要是找到inode
//这个函数的参数是fsnotify_mark_connector结构体，那就去找这个内容
voidfsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
{
if (!conn)
return;
//看一下#3这个栈，rdi和(r12+0x38)都是fsnotify_mark_connector结构体地址
crash> dis -lr ffffffffa0d22d1e |tail -5
0xffffffffa0d22d0e <fsnotify_add_mark_locked+798>: je 0xffffffffa0d22bd7 <fsnotify_add_mark_locked+487>
/build/linux-xiWpQr/linux-5.4.0/fs/notify/mark.c: 664
0xffffffffa0d22d14 <fsnotify_add_mark_locked+804>: mov 0x38(%r12),%rdi
0xffffffffa0d22d19 <fsnotify_add_mark_locked+809>: callq 0xffffffffa0d226b0 <fsnotify_recalc_mask>
0xffffffffa0d22d1e <fsnotify_add_mark_locked+814>: jmpq 0xffffffffa0d22bd7 <fsnotify_add_mark_locked+487>
//再看一下#2这个栈，这里把r12入栈了，压在了第二个栈
crash> dis -lr ffffffffa0d226f8 |head -10
/build/linux-xiWpQr/linux-5.4.0/fs/notify/mark.c: 142
0xffffffffa0d226b0 <fsnotify_recalc_mask>: nopl 0x0(%rax,%rax,1) [FTRACE NOP]
/build/linux-xiWpQr/linux-5.4.0/fs/notify/mark.c: 143
0xffffffffa0d226b5 <fsnotify_recalc_mask+5>: test %rdi,%rdi
0xffffffffa0d226b8 <fsnotify_recalc_mask+8>: je 0xffffffffa0d226fc <fsnotify_recalc_mask+76>
0xffffffffa0d226ba <fsnotify_recalc_mask+10>: push %rbp
0xffffffffa0d226bb <fsnotify_recalc_mask+11>: mov %rsp,%rbp
0xffffffffa0d226be <fsnotify_recalc_mask+14>: push %r12
0xffffffffa0d226c0 <fsnotify_recalc_mask+16>: mov %rdi,%r12
/build/linux-xiWpQr/linux-5.4.0/include/linux/spinlock.h: 338
//看一下#2的栈内容，从后往前找到第二个入栈的参数，就是ffff93d408b8c0f0，也就是这是r12的内容
crash> rd ffffabd947c7fe18 -e ffffabd947c7fe30
ffffabd947c7fe18: ffffffffa0d226f8 ffff93d408b8c0f0 .&..............
ffffabd947c7fe28: ffffabd947c7fe88 ...G....
//然后计算一下fsnotify_mark_connector结构体的地址，偏移0x38，然后读出的内容ffff93d42b4d4660就是这个结构体的地址
//然后再找出obj的值
crash> p/x 0xffff93d408b8c0f0+0x38
$7 = 0xffff93d408b8c128
crash> rd 0xffff93d408b8c128
ffff93d408b8c128: ffff93d42b4d4660 `FM+....
crash> struct fsnotify_mark_connector.obj ffff93d42b4d4660
obj = 0xffff936b8b4c97d8
//这里的fsnotify_mark_connector结构体的地址应该就是inode中的i_fsnotify_marks的指针
//所以找到i_fsnotify_marks在inode中的偏移，就知道inode的地址了，所以inode的地址就是0xffff936b8b4c95a0
crash> struct inode.i_fsnotify_marks -ox
struct inode {
[0x238] struct fsnotify_mark_connector *i_fsnotify_marks;
}
crash> px 0xffff936b8b4c97d8-0x238
$5 = 0xffff936b8b4c95a0
//identry链表链的就是dentry的d_u.d_alias
//可以看一下hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
//所以找到du结构体的偏移，减去这个偏移即是dentry结构体的地址，所以dentry结构体地址就是0xffff92fded655140
crash> struct inode.i_dentry 0xffff936b8b4c95a0
i_dentry = {
first = 0xffff92fded6551f0
}
crash> struct dentry.d_u -ox
struct dentry {
union {
[0xb0] } d_u;
}
crash> px 0xffff92fded6551f0-0xb0
$6 = 0xffff92fded655140
//然后就可以查看这个file了
crash> files -d 0xffff92fded655140
DENTRY INODE SUPERBLK TYPE PATH
ffff92fded655140 ffff936b8b4c95a0 ffff93dad0f17000 DIR /run/containerd/xxx/xxx/xxx/xxx/xxx/xxx/xxx/xxx/xxx/xxx
//然后看一下，这个dentry下的subdirs
crash> struct dentry.d_subdirs ffff92fded655140
d_subdirs = {
next = 0xffff930e939c1950,
prev = 0xffff92ee96445a10
}
crash> list0xffff930e939c1950
ffff930e939c1950
ffff92ede851ddd0
ffff92f0a7fb7590
ffff92fbad973710
ffff92fbad9728d0
ffff92fbad972e10
ffff92fbd0ac8090
ffff92fbd0ac85d0
ffff92fbd0ac9890
ffff92fbd0ac9290
ffff92fbd0ac8750
.......
crash> list0xffff930e939c1950 |wc -l
//内容太多 卡住language-c复制代码
所以可以确定是https://access.redhat.com/solutions/7095274这个问题
继续查看，上游可能已经给出了修复172e422ffea2(“fsnotify: clear PARENT_WATCHED flags lazily”)，24年提的，可能现在还没用到5.4 stable中去