Skip to content

Crash on stress test #2522

@const-t

Description

@const-t

Got crash on VM with 14288 MB of memory while stress testing with h2load. Can be reproduced on 6.12.12 kernel on 5.10.35 can't reproduce the issue. Used config as simple as possible. Cache disabled.

h2load -c1000 -m100 -t4 -D40 https://ubuntu/10.png
10.png - 100kb file.

[   22.502441] ------------[ cut here ]------------
[   22.502863] kernel BUG at /home/constantine/projects/tempesta/fw/http_msg.c:1226!
[   22.503498] Oops: invalid opcode: 0000 [#1] PREEMPT SMP NOPTI
[   22.503986] CPU: 1 UID: 118 PID: 3023 Comm: SystemLogFlush Tainted: G           OE      6.12.12+ #168
[   22.504760] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[   22.505361] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS Arch Linux 1.17.0-1-1 04/01/2014
[   22.506130] RIP: 0010:__tfw_http_msg_expand_from_pool+0x545/0x6e0 [tempesta_fw]
[   22.506759] Code: 74 29 97 d8 00 00 00 48 8b 7c 24 38 01 50 70 01 50 74 01 90 d8 00 00 00 44 89 c2 48 89 07 49 8b bf 9c 00 00 00 e9 05 fd ff ff <0f> 0b 0f 0b 49 8b 87 9c 00 00 00 41 8d 51 ff e9 42 fd ff ff 41 8b
[   22.508305] RSP: 0018:ffffb8f240100790 EFLAGS: 00010206
[   22.508750] RAX: ffff9ac555fbf000 RBX: ffff9ac54ef6e150 RCX: ffffffffc0ed0120
[   22.509357] RDX: ffffb8f24010082c RSI: ffffb8f240100920 RDI: ffff9ac54ef6e020
[   22.509939] RBP: ffffb8f240100818 R08: ffff9ac515231010 R09: 0000000000000003
[   22.510524] R10: ffffefdac64d3200 R11: 0000000000000000 R12: 00000000ffffff88
[   22.511121] R13: 0000000000000001 R14: 0000000000000008 R15: ffff9ac50c60e000
[   22.511721] FS:  00007f06afbfe6c0(0000) GS:ffff9ac7aec80000(0000) knlGS:0000000000000000
[   22.512399] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   22.512887] CR2: 00007f06b7e82000 CR3: 000000010be54000 CR4: 0000000000750ef0
[   22.513490] PKRU: 55555554
[   22.513730] Call Trace:
[   22.513954]  <IRQ>
[   22.514146]  ? __die_body.cold+0x19/0x34
[   22.514507]  ? die+0x2e/0x60
[   22.514774]  ? do_trap+0xca/0x120
[   22.515079]  ? do_error_trap+0x6a/0xa0
[   22.515542]  ? __tfw_http_msg_expand_from_pool+0x545/0x6e0 [tempesta_fw]
[   22.516251]  ? exc_invalid_op+0x50/0x80
[   22.516700]  ? __tfw_http_msg_expand_from_pool+0x545/0x6e0 [tempesta_fw]
[   22.517381]  ? asm_exc_invalid_op+0x1a/0x20
[   22.517865]  ? __pfx_memcpy_fast+0x20/0x20 [tempesta_lib]
[   22.518429]  ? __tfw_http_msg_expand_from_pool+0x545/0x6e0 [tempesta_fw]
[   22.519109]  ? virtqueue_add_split+0x193/0x7c0
[   22.519608]  ? __pfx_memcpy_fast+0x20/0x20 [tempesta_lib]
[   22.520167]  tfw_h2_msg_expand_from_pool+0x38/0x60 [tempesta_fw]
[   22.520783]  __tfw_hpack_encode+0x70a/0xf00 [tempesta_fw]
[   22.521347]  ? virtqueue_add_outbuf+0x4a/0x60
[   22.521816]  ? start_xmit+0x48a/0x700 [virtio_net]
[   22.522325]  tfw_h2_resp_status_write+0x10a/0x220 [tempesta_fw]
[   22.522930]  tfw_h2_resp_encode_headers+0x141/0x480 [tempesta_fw]
[   22.523550]  tfw_h2_stream_xmit_prepare_resp+0x8f/0x340 [tempesta_fw]
[   22.524200]  tfw_h2_make_frames+0x135/0x800 [tempesta_fw]
[   22.524759]  tfw_sk_fill_write_queue+0x13b/0x1a0 [tempesta_fw]
[   22.525362]  tcp_push_pending_frames+0x33/0x160
[   22.525864]  tcp_rcv_established+0x263/0x6e0
[   22.526337]  tcp_v4_do_rcv+0x161/0x2c0
[   22.526757]  tcp_v4_rcv+0x1115/0x14e0
[   22.527171]  ? raw_local_deliver+0xd6/0x280
[   22.527622]  ip_protocol_deliver_rcu+0x36/0x300
[   22.528118]  ip_local_deliver_finish+0x76/0xa0
[   22.528593]  ip_local_deliver+0x68/0x120
[   22.529018]  ? __pfx_ip_local_deliver_finish+0x20/0x20
[   22.529546]  __netif_receive_skb_one_core+0x87/0xa0
[   22.530069]  process_backlog+0x87/0x140
[   22.530487]  __napi_poll+0x28/0x160
[   22.530874]  net_rx_action+0x3a7/0x420
[   22.531284]  handle_softirqs+0xdd/0x2a0
[   22.531705]  __irq_exit_rcu+0x5f/0x80
[   22.532113]  common_interrupt+0x85/0xa0
[   22.532529]  </IRQ>
[   22.532792]  <TASK>
[   22.533063]  asm_common_interrupt+0x26/0x40
[   22.533503] RIP: 0010:__pv_queued_spin_lock_slowpath+0x2b0/0x380
[   22.534099] Code: c0 72 03 00 8d 42 ff 48 98 48 03 2c c5 00 de 2b a2 4c 89 75 00 b8 00 80 00 00 eb 13 84 c0 75 08 0f b6 55 14 84 d2 75 36 f3 90 <83> e8 01 74 2f 41 8b 56 08 85 d2 74 e5 41 8b 46 08 85 c0 75 0a f3
[   22.535788] RSP: 0018:ffffb8f243977990 EFLAGS: 00000282
[   22.536310] RAX: 00000000000036ef RBX: 0000000000080000 RCX: 0000000000000001
[   22.536979] RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffff9ac7aecb72d4
[   22.537657] RBP: ffff9ac7aec372c0 R08: ffff9ac4c84e1888 R09: 0000000000000001
[   22.538338] R10: ffff9ac4c4bdb000 R11: 0000000000000001 R12: 0000000000000001
[   22.539013] R13: ffff9ac4c16c6400 R14: ffff9ac7aecb72c0 R15: 0000000000000000
[   22.539712]  ? __ext4_journal_get_write_access+0x42/0x160
[   22.540262]  _raw_spin_lock+0x29/0x40
[   22.540657]  ext4_mb_mark_context+0x2f7/0x400
[   22.541128]  ext4_mb_mark_diskspace_used+0xd2/0x1c0
[   22.541623]  ext4_mb_new_blocks+0x176/0xe60
[   22.542099]  ? ext4_find_extent+0x352/0x3a0
[   22.542536]  ? ext4_find_extent+0x37c/0x3a0
[   22.542967]  ext4_ext_map_blocks+0x8be/0x18c0
[   22.543429]  ? ext4_ext_map_blocks+0x339/0x18c0
[   22.543891]  ? __ext4_handle_dirty_metadata+0x60/0x1a0
[   22.544436]  ext4_map_blocks+0x16a/0x520
[   22.544858]  ext4_getblk+0xa3/0x240
[   22.545241]  ext4_bread+0xf/0x80
[   22.545596]  ext4_append+0xa9/0x1c0
[   22.545973]  ext4_init_new_dir+0xd6/0x180
[   22.546427]  ext4_mkdir+0x112/0x340
[   22.546800]  vfs_mkdir+0x191/0x260
[   22.547188]  do_mkdirat+0x14d/0x180
[   22.547568]  __x64_sys_mkdir+0x46/0x80
[   22.547961]  do_syscall_64+0x4f/0x120
[   22.548349]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
[   22.548837] RIP: 0033:0x7f084351b05b
[   22.549217] Code: 0f 1e fa 48 89 f2 b9 00 01 00 00 48 89 fe bf 9c ff ff ff e9 97 c3 ff ff 0f 1f 80 00 00 00 00 f3 0f 1e fa b8 53 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 05 c3 0f 1f 40 00 48 8b 15 89 7d 0e 00 f7 d8
[   22.550906] RSP: 002b:00007f06afbf3898 EFLAGS: 00000282 ORIG_RAX: 0000000000000053
[   22.551619] RAX: ffffffffffffffda RBX: 00007f06afbf3ae0 RCX: 00007f084351b05b
[   22.552299] RDX: 0000000000000002 RSI: 00000000000001ff RDI: 00007f06a25bb7a0
[   22.552958] RBP: 00007f06a25bb7a0 R08: 00007f06a25bb7bc R09: 0000000000000043
[   22.553643] R10: fffffffffffffffe R11: 0000000000000282 R12: 00007f06b19b4ac0
[   22.554331] R13: 0000000000000042 R14: 00007f06afbf3ae0 R15: 0000000000000000
[   22.555012]  </TASK>
[   22.555288] Modules linked in: tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) xt_conntrack nft_chain_nat xt_MASQUERADE nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 bridge stp llc xfrm_user xfrm_algo xt_addrtype nft_compat nf_tables overlay intel_rapl_msr intel_rapl_common cfg80211 snd_hda_codec_generic joydev kvm_amd snd_hda_intel snd_intel_dspcfg ccp hid_generic binfmt_misc kvm snd_intel_sdw_acpi snd_hda_codec snd_hda_core crct10dif_pclmul snd_hwdep crc32_pclmul ghash_clmulni_intel sha512_ssse3 snd_pcm i2c_i801 ahci usbhid i2c_mux snd_timer sha256_ssse3 libahci hid snd i2c_smbus soundcore lpc_ich virtio_scsi virtio_net virtiofs net_failover failover sch_fq_codel dm_multipath efi_pstore nfnetlink dmi_sysfs qemu_fw_cfg virtio_rng ip_tables x_tables autofs4 raid10 raid456 libcrc32c async_raid6_recov async_memcpy async_pq async_xor xor async_tx raid6_pq raid1 raid0 qxl drm_ttm_helper ttm drm_kms_helper input_leds psmouse drm serio_raw virtio_blk mac_hid aesni_intel crypto_simd cryptd
[   22.563267] ---[ end trace 0000000000000000 ]---
[   22.563770] RIP: 0010:__tfw_http_msg_expand_from_pool+0x545/0x6e0 [tempesta_fw]
[   22.564514] Code: 74 29 97 d8 00 00 00 48 8b 7c 24 38 01 50 70 01 50 74 01 90 d8 00 00 00 44 89 c2 48 89 07 49 8b bf 9c 00 00 00 e9 05 fd ff ff <0f> 0b 0f 0b 49 8b 87 9c 00 00 00 41 8d 51 ff e9 42 fd ff ff 41 8b
[   22.566264] RSP: 0018:ffffb8f240100790 EFLAGS: 00010206
[   22.566807] RAX: ffff9ac555fbf000 RBX: ffff9ac54ef6e150 RCX: ffffffffc0ed0120
[   22.567512] RDX: ffffb8f24010082c RSI: ffffb8f240100920 RDI: ffff9ac54ef6e020
[   22.568231] RBP: ffffb8f240100818 R08: ffff9ac515231010 R09: 0000000000000003
[   22.568932] R10: ffffefdac64d3200 R11: 0000000000000000 R12: 00000000ffffff88
[   22.569636] R13: 0000000000000001 R14: 0000000000000008 R15: ffff9ac50c60e000
[   22.570342] FS:  00007f06afbfe6c0(0000) GS:ffff9ac7aec80000(0000) knlGS:0000000000000000
[   22.571129] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   22.571725] CR2: 00007f06b7e82000 CR3: 000000010be54000 CR4: 0000000000750ef0
[   22.572450] PKRU: 55555554
[   22.572801] Kernel panic - not syncing: Fatal exception in interrupt

Metadata

Metadata

Assignees

Type

No type

Projects

No projects

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions