Fix Sema unreachable when shfting by vector containing undef elems

[Justus2308]

Closes #24392

I added some missing checks for undef values before using compareHetero and getUnsignedInt to avoid hitting an unreachable prong.
This also enables a small AIR optimization basically for free that replaces a vec that only contains undef elems with a single undef value and returns early from zirShl/zirShr, skipping some unnecessary logic and emitting a shift instruction:

export fn foobar(x: @Vector(4, u8)) @Vector(4, u8) {
    const a = x << @as(@Vector(4, u3), .{ undefined, undefined, undefined, undefined });
    return a;
}

old AIR

# Begin Function AIR: repro.foobar:
# Total AIR+Liveness bytes: 206B
# AIR Instructions:         6 (54B)
# AIR Extra Data:           8 (32B)
# Liveness tomb_bits:       8B
# Liveness Extra Data:      0 (0B)
# Liveness special table:   0 (0B)
  %0 = arg(@Vector(4, u8), 0)
  %1!= save_err_return_trace_index()
  %2!= dbg_stmt(2:14)
  %3 = shl(%0!, <@Vector(4, u3), .{ undefined, undefined, undefined, undefined }>)
  %4!= dbg_stmt(2:5)
  %5!= ret_safe(%3!)
# End Function AIR: repro.foobar

new AIR

# Begin Function AIR: repro.foobar:
# Total AIR+Liveness bytes: 180B
# AIR Instructions:         4 (36B)
# AIR Extra Data:           6 (24B)
# Liveness tomb_bits:       8B
# Liveness Extra Data:      0 (0B)
# Liveness special table:   0 (0B)
  %0!= arg(@Vector(4, u8), 0)
  %1!= save_err_return_trace_index()
  %2!= dbg_stmt(2:5)
  %3!= ret_safe(<@Vector(4, u8), undefined>)
# End Function AIR: repro.foobar

old LLVM IR

; Function Attrs: nounwind sspstrong uwtable
define dso_local <4 x i8> @foobar(<4 x i8> %0) #0 !dbg !195 {
Entry:
  %1 = alloca <4 x i8>, align 4
  %2 = alloca <4 x i8>, align 4
  store <4 x i8> %0, ptr %2, align 4, !dbg !196
  call void @llvm.dbg.declare(metadata ptr %2, metadata !197, metadata !DIExpression()), !dbg !196
  %3 = zext <4 x i3> <i3 undef, i3 undef, i3 undef, i3 undef> to <4 x i8>, !dbg !198
  %4 = shl <4 x i8> %0, %3, !dbg !198
  store <4 x i8> %4, ptr %1, align 4, !dbg !198
  call void @llvm.dbg.declare(metadata ptr %1, metadata !199, metadata !DIExpression()), !dbg !198
  ret <4 x i8> %4, !dbg !200
}

ReleaseFast:

; Function Attrs: nounwind uwtable
define dso_local <16 x i8> @foobar(<16 x i8> %0) #0 !dbg !182 {
Entry:
  call void @llvm.dbg.value(metadata <16 x i8> %0, metadata !184, metadata !DIExpression()), !dbg !183
  %1 = zext <16 x i3> <i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef, i3 undef> to <16 x i8>, !dbg !185
  %2 = shl <16 x i8> %0, %1, !dbg !185
  call void @llvm.dbg.value(metadata <16 x i8> %2, metadata !186, metadata !DIExpression()), !dbg !185
  ret <16 x i8> %2, !dbg !187
}

new LLVM IR

; Function Attrs: nounwind sspstrong uwtable
define dso_local <4 x i8> @foobar(<4 x i8> %0) #0 !dbg !194 {
Entry:
  %1 = alloca <4 x i8>, align (4)
  %2 = alloca <4 x i8>, align (4)
  %3 = alloca <4 x i8>, align (4)
  store <4 x i8> %0, ptr %3, align (4), !dbg !195
  call void @llvm.dbg.declare(metadata ptr %3, metadata !196, metadata !DIExpression()), !dbg !195
  store <4 x i8> undef, ptr %2, align (4), !dbg !197
  call void @llvm.dbg.declare(metadata ptr %2, metadata !198, metadata !DIExpression()), !dbg !197
  call void @llvm.memset.p0.i64(ptr align (4) %1, i8 -86, i64 4, i1 false), !dbg !199
  %4 = load <4 x i8>, ptr %1, align (4), !dbg !199
  ret <4 x i8> %4, !dbg !199
}

ReleaseFast:

; Function Attrs: nounwind uwtable
define dso_local <16 x i8> @foobar(<16 x i8> %0) #0 !dbg !181 {
Entry:
  call void @llvm.dbg.value(metadata <16 x i8> %0, metadata !183, metadata !DIExpression()), !dbg !182
  call void @llvm.dbg.value(metadata <16 x i8> undef, metadata !185, metadata !DIExpression()), !dbg !184
  ret <16 x i8> undef, !dbg !186
}

[Justus2308]

I cannot reproduce the CI failure locally from a M1 MacBook Pro using

zig build test-behavior -fqemu -Duse-llvm -Dtest-target-filter="loongarch64-linux-musl" --seed 0xa2eb50f3 --summary all

I'm using qemu 10.0.2 from homebrew, could this be something that can only be reprod with ziglang/static-qemu?

EDIT: I was able to retrieve a loongarch64 coredump by running static-qemu qemu-loongarch64 on utm debian 11

[alexrp]

It is quite possible that this failure is not actually your fault. Did you find anything interesting in the coredump?

[alexrp]

Managed to repro that failure locally, will investigate.

[alexrp]

Going with #24407 for now to get CI stable.

[Justus2308]

Thanks for the help! I haven't had time to look into the coredump yet but I'll let you know if I find anything.